From 50fdb15fac0f99efe51ec26f0e2e71388a3f350c Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:58:43 -0700 Subject: [PATCH 01/44] test(runtime): expose Hermes activation transport bound Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../docker-state-mutation.test.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 03a04896532..ef59b0ba807 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -55,6 +55,34 @@ afterEach(() => { }); describe("Docker runtime-provider state mutation surface", () => { + it("keeps the detached activation broker alive through retained-fence recovery (#10155)", () => { + const controller = path.join( + import.meta.dirname, + "../../../../scripts/runtime-state-mutation-control.py", + ); + const activationWindowSeconds = Number( + execFileSync( + "python3", + [ + "-I", + "-c", + "import importlib.util,sys;spec=importlib.util.spec_from_file_location('control',sys.argv[1]);control=importlib.util.module_from_spec(spec);sys.modules[spec.name]=control;spec.loader.exec_module(control);print(control.ACTIVATION_SECONDS)", + controller, + ], + { encoding: "utf8", timeout: 5_000 }, + ).trim(), + ); + const brokerTimeouts = JSON.parse( + /^TIMEOUTS = (\{[^\n]+\})$/mu.exec( + DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE, + )?.[1] ?? "null", + ) as Record | null; + + expect(activationWindowSeconds).toBeGreaterThan(0); + expect(brokerTimeouts?.activate).toEqual(expect.any(Number)); + expect(brokerTimeouts?.activate as number).toBeGreaterThan(activationWindowSeconds * 3); + }); + it("preserves safe broker diagnostics after request validation", () => { const definitionsEnd = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.indexOf( "\nhelper = sys.argv[1]\n", From 13973551d8549fec44723442b92d70f764e9c30a Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:35:35 -0700 Subject: [PATCH 02/44] fix(runtime): preserve activation transport response Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../docker-state-mutation.test.ts | 38 ++++++++++++++++++- .../runtime-provider/docker-state-mutation.ts | 19 ++++++---- test/helpers/docker-state-mutation-harness.ts | 8 ++-- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index ef59b0ba807..7b0df794a24 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -127,6 +127,40 @@ print(json.dumps({ }); }); + it("keeps the signal replay inside one activation broker deadline (#10155)", () => { + const definitionsEnd = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.indexOf( + "\nhelper = sys.argv[1]\n", + ); + const definitions = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.slice( + 0, + definitionsEnd, + ); + const probe = `${definitions} +import types +helper = "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py" +os.lstat = lambda _path: types.SimpleNamespace( + st_mode=stat.S_IFREG | 0o444, st_uid=0, st_gid=0) +ticks = iter((100.0, 100.0, 325.0)) +time.monotonic = lambda: next(ticks) +timeouts = [] +def fake_run(*_args, **kwargs): + timeouts.append(kwargs["timeout"]) + return types.SimpleNamespace(returncode=-9 if len(timeouts) == 1 else 0) +subprocess.run = fake_run +run_helper("activate", b"{}\\n") +print(json.dumps(timeouts)) +`; + + expect( + JSON.parse( + execFileSync("python3", ["-I", "-c", probe], { + encoding: "utf8", + timeout: 5_000, + }), + ), + ).toEqual([480, 255]); + }); + it("uses one harness-owned absolute Docker executable", () => { const runtime = harness(); runtime.authority.engine.capture(["version"]); @@ -382,8 +416,8 @@ describe("Docker state mutation owner", () => { ["acquire", 30_000], ["assert", 30_000], ["rollback", 15 * 60_000], - ["activate", 5 * 60_000], - ["activate", 5 * 60_000], + ["activate", 8 * 60_000], + ["activate", 8 * 60_000], ["release", 5 * 60_000], ]); const acquireRequest = JSON.parse(helperCalls[0]?.[3]?.toString("utf8") ?? "null"); diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index 203c4e653c9..85f97b18447 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -54,7 +54,11 @@ const SUPPORTED_STATE_ROOT = "/sandbox/.hermes"; const HELPER_PYTHON_PATH = "/opt/hermes/.venv/bin/python3"; const HELPER_PATH = "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py"; const HELPER_FAST_TIMEOUT_MS = 30_000; -const HELPER_ACTIVATION_TIMEOUT_MS = 5 * 60_000; +// Retained activation recovery can consume three consecutive 150-second +// controller windows: retry acknowledgement, startup checkpoint, and live +// service verification. Keep the broker outside that bound and below the +// existing 15-minute Shields state-mutation guard. +const HELPER_ACTIVATION_TIMEOUT_MS = 8 * 60_000; export const DOCKER_STATE_MUTATION_GUARD_TIMEOUT_MS = 15 * 60_000; const INSPECT_TIMEOUT_MS = 15_000; const SUPERVISOR_SIGNAL_TIMEOUT_MS = 15_000; @@ -90,7 +94,7 @@ import time ROOT = "/run/nemoclaw/runtime-state-mutation" MAXIMUM = 128 * 1024 -TIMEOUTS = {"acquire": 30, "assert": 30, "publish": 900, "recover": 900, "rollback": 900, "activate": 300, "release": 300} +TIMEOUTS = {"acquire": 30, "assert": 30, "publish": 900, "recover": 900, "rollback": 900, "activate": ${HELPER_ACTIVATION_TIMEOUT_MS / 1000}, "release": 300} IDENTITY = re.compile(r"[a-f0-9]{64}\Z") INCOMING = re.compile(r"([a-f0-9]{64})\.(acquire|assert|publish|recover|rollback|activate|release)\.incoming\Z") PUBLICATION_SETTLE_SECONDS = 5 @@ -227,16 +231,15 @@ def run_helper(action, request): if (not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0 or metadata.st_gid != 0 or stat.S_IMODE(metadata.st_mode) & 0o022): fail("helper-file-invalid") - completed = None - for attempt in range(2): + deadline = time.monotonic() + TIMEOUTS[action] + for _ in range(2): completed = subprocess.run([sys.executable, "-I", helper, action], input=request, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=TIMEOUTS[action], check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=max(0.001, deadline - time.monotonic()), check=False, start_new_session=True) if completed.returncode >= 0: return completed - # Every helper action is transaction-bound and idempotent. Replay only - # a signal-terminated invocation once; ordinary nonzero exits remain - # authoritative and are never retried. + # Replay one signal exit inside this action's deadline. return completed helper = sys.argv[1] diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index 81c7c476be4..72656db10fc 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -341,9 +341,11 @@ function createContainerStateMutationHarness( const helperTimeout = action === "acquire" || action === "assert" ? 30_000 - : action === "activate" || action === "release" - ? 5 * 60_000 - : 15 * 60_000; + : action === "activate" + ? 8 * 60_000 + : action === "release" + ? 5 * 60_000 + : 15 * 60_000; let helperResult = capture( "docker", [ From 8c3518ea0867f1391b2adfc5f421e05dfd644f50 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:49:40 -0700 Subject: [PATCH 03/44] fix(runtime): keep release transport bound Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../onboard/runtime-provider/docker-state-mutation.test.ts | 1 + src/lib/onboard/runtime-provider/docker-state-mutation.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 7b0df794a24..de428890ec0 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -81,6 +81,7 @@ describe("Docker runtime-provider state mutation surface", () => { expect(activationWindowSeconds).toBeGreaterThan(0); expect(brokerTimeouts?.activate).toEqual(expect.any(Number)); expect(brokerTimeouts?.activate as number).toBeGreaterThan(activationWindowSeconds * 3); + expect(brokerTimeouts?.release).toBe(300); }); it("preserves safe broker diagnostics after request validation", () => { diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index 85f97b18447..c50c4c130fa 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -59,6 +59,7 @@ const HELPER_FAST_TIMEOUT_MS = 30_000; // service verification. Keep the broker outside that bound and below the // existing 15-minute Shields state-mutation guard. const HELPER_ACTIVATION_TIMEOUT_MS = 8 * 60_000; +const HELPER_RELEASE_TIMEOUT_MS = 5 * 60_000; export const DOCKER_STATE_MUTATION_GUARD_TIMEOUT_MS = 15 * 60_000; const INSPECT_TIMEOUT_MS = 15_000; const SUPERVISOR_SIGNAL_TIMEOUT_MS = 15_000; @@ -370,8 +371,9 @@ function helperTimeoutMs(action: HelperAction): number { case "rollback": return DOCKER_STATE_MUTATION_GUARD_TIMEOUT_MS; case "activate": - case "release": return HELPER_ACTIVATION_TIMEOUT_MS; + case "release": + return HELPER_RELEASE_TIMEOUT_MS; case "acquire": case "assert": return HELPER_FAST_TIMEOUT_MS; From e50c63eee00eeb13455aecad9e08fed60f53d9f1 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:57:04 -0700 Subject: [PATCH 04/44] test(runtime): parse activation window portably Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../runtime-provider/docker-state-mutation.test.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index de428890ec0..735a909c92b 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -60,18 +60,10 @@ describe("Docker runtime-provider state mutation surface", () => { import.meta.dirname, "../../../../scripts/runtime-state-mutation-control.py", ); - const activationWindowSeconds = Number( - execFileSync( - "python3", - [ - "-I", - "-c", - "import importlib.util,sys;spec=importlib.util.spec_from_file_location('control',sys.argv[1]);control=importlib.util.module_from_spec(spec);sys.modules[spec.name]=control;spec.loader.exec_module(control);print(control.ACTIVATION_SECONDS)", - controller, - ], - { encoding: "utf8", timeout: 5_000 }, - ).trim(), + const activationWindow = /^ACTIVATION_SECONDS = ([1-9][0-9]*(?:\.[0-9]+)?)$/mu.exec( + fs.readFileSync(controller, "utf8"), ); + const activationWindowSeconds = Number(activationWindow?.[1] ?? Number.NaN); const brokerTimeouts = JSON.parse( /^TIMEOUTS = (\{[^\n]+\})$/mu.exec( DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE, From efc5b2afd6d3f5ad11fb7f70fadc0d237e6b9be9 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:17:25 -0700 Subject: [PATCH 05/44] fix(runtime): acknowledge Hermes activation release Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- agents/hermes/start.sh | 38 ++++- scripts/runtime-state-mutation-control.py | 108 ++++++++++++- .../runtime-state-mutation-startup-gate.py | 84 +++++++++- .../docker-state-mutation.test.ts | 104 ++++++++++++- .../runtime-provider/docker-state-mutation.ts | 26 +++- test/helpers/docker-state-mutation-harness.ts | 2 +- .../runtime-state-mutation-control.test.ts | 1 + ...me-state-mutation-hermes-publisher.test.ts | 12 ++ ...runtime-state-mutation-release-ack.test.ts | 143 ++++++++++++++++++ ...untime-state-mutation-startup-gate.test.ts | 28 +++- 10 files changed, 526 insertions(+), 20 deletions(-) create mode 100644 test/state/runtime-state-mutation-release-ack.test.ts diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index e96c2ad6555..ca51e75ee40 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -30,10 +30,13 @@ NEMOCLAW_RUNTIME_STATE_MUTATION_RETRY_ARGV=("$@") readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON="/opt/hermes/.venv/bin/python3" readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER="/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py" readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV="/usr/bin/setpriv" +readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV="/usr/bin/mv" +readonly NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT="/run/nemoclaw/runtime-state-mutation-startup" if [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" ] \ || [ ! -f "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \ || [ -L "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \ + || [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV" ] \ || { [ "$EUID" -eq 0 ] && [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" ]; }; then printf '%s\n' '[SECURITY] Required runtime state mutation startup gate is unavailable.' >&2 exit 1 @@ -52,6 +55,34 @@ nemoclaw_runtime_state_mutation_gate() { "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" "$action" >/dev/null } +nemoclaw_runtime_state_mutation_acknowledge_release() { + local nonce pending final + if [ "$EUID" -eq 0 ]; then + nonce="$("$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" \ + --reuid=sandbox --regid=sandbox --init-groups -- \ + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge)" || return 1 + else + nonce="$("$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge)" || return 1 + fi + if [ "${#nonce}" -ne 64 ]; then + return 1 + fi + case "$nonce" in + *[!0-9a-f]*) return 1 ;; + esac + pending="${NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT}/${nonce}/.release-ack.json.pending" + final="${NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT}/${nonce}/release-ack.json" + if [ -f "$final" ] && [ ! -L "$final" ]; then + return 0 + fi + if [ ! -f "$pending" ] || [ -L "$pending" ]; then + return 1 + fi + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV" -f -- "$pending" "$final" +} + nemoclaw_runtime_state_mutation_retry_exec() { local status if nemoclaw_runtime_state_mutation_gate restart; then @@ -106,7 +137,12 @@ nemoclaw_runtime_state_mutation_checkpoint() { fi kill -STOP "$$" if nemoclaw_runtime_state_mutation_gate resume; then - return 0 + if nemoclaw_runtime_state_mutation_acknowledge_release; then + return 0 + fi + printf '%s\n' '[SECURITY] Runtime state mutation release acknowledgement failed; holding startup.' >&2 + kill -STOP "$$" + return 1 else status=$? fi diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 7d0015e8a8a..a1b70aa04e5 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -92,6 +92,8 @@ ACTIVATION_CLEANUP_NAME = "activation-cleanup.json" STARTUP_CANDIDATE_NAME = "startup-complete.json" STARTUP_RETRY_ACK_NAME = "retry-ack.json" +STARTUP_RELEASE_ACK_NAME = "release-ack.json" +STARTUP_RELEASE_ACK_PENDING_NAME = ".release-ack.json.pending" RELEASED_RECEIPT_NAME = "released.json" PUBLISHER_MODULE_PATH = ( "/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py" @@ -103,6 +105,7 @@ ACTIVATION_CLEANUP_PROTOCOL = "nemoclaw-runtime-state-mutation-activation-cleanup-v1" STARTUP_CANDIDATE_PROTOCOL = "nemoclaw-runtime-state-mutation-startup-complete-v1" STARTUP_RETRY_ACK_PROTOCOL = "nemoclaw-runtime-state-mutation-retry-ack-v1" +STARTUP_RELEASE_ACK_PROTOCOL = "nemoclaw-runtime-state-mutation-release-ack-v1" OPENSHELL_ARGV0 = b"/opt/openshell/bin/openshell-sandbox" NEMOCLAW_START_PATH = b"/usr/local/bin/nemoclaw-start" BASH_ARGV0 = (b"bash", b"/bin/bash", b"/usr/bin/bash") @@ -919,6 +922,14 @@ def _parse_request(action: Action, raw: bytes) -> Request: "retrySha256", "start", ) +STARTUP_RELEASE_ACK_KEYS = ( + "schemaVersion", + "protocol", + "transactionId", + "nonce", + "releaseSha256", + "start", +) def _process_command_sha256(command: tuple[bytes, ...]) -> str: @@ -1107,6 +1118,19 @@ def _startup_retry_ack_payload( } +def _startup_release_ack_payload( + marker: dict[str, object], fence: FenceProof, release_payload: bytes +) -> dict[str, object]: + return { + "schemaVersion": SCHEMA_VERSION, + "protocol": STARTUP_RELEASE_ACK_PROTOCOL, + "transactionId": marker["transactionId"], + "nonce": marker["nonce"], + "releaseSha256": _sha256(release_payload), + "start": _process_reference_payload(fence.start), + } + + def _marker_payload( request: AcquireRequest, phase: Phase, @@ -2022,6 +2046,77 @@ def _wait_for_startup_retry_ack( time.sleep(min(POLL_SECONDS, remaining)) +def _read_startup_release_ack( + marker: dict[str, object], fence: FenceProof, release_payload: bytes +) -> dict[str, object] | None: + opened = _open_startup_candidate_directory(marker, create=False) + if opened is None: + return None + root_fd, directory_fd = opened + sandbox_uid, sandbox_gid = _sandbox_account() + try: + try: + fd = os.open( + STARTUP_RELEASE_ACK_NAME, + os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK, + dir_fd=directory_fd, + ) + except FileNotFoundError: + return None + except OSError: + _fail("activation-release-ack-invalid") + try: + before = os.fstat(fd) + payload = os.read(fd, MAX_MARKER_BYTES + 1) + after = os.fstat(fd) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_uid != sandbox_uid + or before.st_gid != sandbox_gid + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_nlink != 1 + or len(payload) > MAX_MARKER_BYTES + or os.read(fd, 1) + or _stable_stat(before) != _stable_stat(after) + ): + _fail("activation-release-ack-invalid") + finally: + os.close(fd) + ack = _exact_keys( + _parse_json(payload, MAX_MARKER_BYTES, "activation-release-ack-invalid"), + STARTUP_RELEASE_ACK_KEYS, + "activation-release-ack-invalid", + ) + expected = _startup_release_ack_payload( + marker, fence, release_payload + ) + if ( + ack["schemaVersion"] != SCHEMA_VERSION + or ack["protocol"] != STARTUP_RELEASE_ACK_PROTOCOL + or payload != _json_bytes(expected) + b"\n" + or not secrets.compare_digest(_json_bytes(ack), _json_bytes(expected)) + ): + _fail("activation-release-ack-invalid") + return expected + finally: + os.close(directory_fd) + os.close(root_fd) + + +def _wait_for_startup_release_ack( + marker: dict[str, object], fence: FenceProof, release_payload: bytes +) -> None: + deadline = time.monotonic() + PROCESS_STATE_SECONDS + while True: + if _read_startup_release_ack(marker, fence, release_payload) is not None: + _recapture_reference(fence.start, "activation-release-identity-drift") + return + remaining = deadline - time.monotonic() + if remaining <= 0: + _fail("activation-release-ack-timeout") + time.sleep(min(POLL_SECONDS, remaining)) + + def _verify_activation_checkpoint( marker: dict[str, object], fence: FenceProof, activation: ActivationProof ) -> None: @@ -2043,9 +2138,16 @@ def _cleanup_startup_candidate_directory(marker: dict[str, object]) -> None: try: for name in os.listdir(directory_fd): if ( - name not in (STARTUP_CANDIDATE_NAME, STARTUP_RETRY_ACK_NAME) + name + not in ( + STARTUP_CANDIDATE_NAME, + STARTUP_RETRY_ACK_NAME, + STARTUP_RELEASE_ACK_NAME, + STARTUP_RELEASE_ACK_PENDING_NAME, + ) and not name.startswith(f".{STARTUP_CANDIDATE_NAME}.") and not name.startswith(f".{STARTUP_RETRY_ACK_NAME}.") + and not name.startswith(f".{STARTUP_RELEASE_ACK_NAME}.") ): _fail("activation-candidate-directory-invalid") try: @@ -3831,6 +3933,9 @@ def _release_activation_hold(durable_fd: int, marker: dict[str, object]) -> None _fail("activation-marker-invalid") _verify_activation_checkpoint(marker, fence, activation) _publish_activation_release(durable_fd, marker, fence, activation) + release_payload = _canonical_protocol_payload( + _activation_release_payload(marker, fence, activation) + ) _prove_fence_shape(fence, str(marker["mountNamespace"])) persistent = set(activation.persistent_pids) for reference in activation.processes: @@ -3841,6 +3946,7 @@ def _release_activation_hold(durable_fd: int, marker: dict[str, object]) -> None _resume_reference(reference) _resume_reference(fence.start) _prove_released_activation(marker, fence, activation) + _wait_for_startup_release_ack(marker, fence, release_payload) def _complete_released_receipt( diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index f7df08f9715..6431c30bd73 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -35,11 +35,14 @@ HANDOFF_ROOT = "/run/nemoclaw/runtime-state-mutation-startup" CANDIDATE_NAME = "startup-complete.json" RETRY_ACK_NAME = "retry-ack.json" +RELEASE_ACK_NAME = "release-ack.json" +RELEASE_ACK_PENDING_NAME = ".release-ack.json.pending" PERMIT_PROTOCOL = "nemoclaw-runtime-state-mutation-activation-permit-v1" RELEASE_PROTOCOL = "nemoclaw-runtime-state-mutation-activation-release-v1" RETRY_PROTOCOL = "nemoclaw-runtime-state-mutation-activation-retry-v1" CANDIDATE_PROTOCOL = "nemoclaw-runtime-state-mutation-startup-complete-v1" RETRY_ACK_PROTOCOL = "nemoclaw-runtime-state-mutation-retry-ack-v1" +RELEASE_ACK_PROTOCOL = "nemoclaw-runtime-state-mutation-release-ack-v1" MAX_FILE_BYTES = 32 * 1024 HEX_64 = re.compile(r"[0-9a-f]{64}\Z") DECIMAL = re.compile(r"(?:0|[1-9][0-9]*)\Z") @@ -615,11 +618,80 @@ def _publish_retry_ack(binding: dict[str, object], retry_payload: bytes) -> None os.close(directory_fd) +def _prepare_release_ack(binding: dict[str, object]) -> str: + directory_fd = _open_absolute_directory( + str(binding["candidateDirectory"]), readable_final=True + ) + try: + metadata = os.fstat(directory_fd) + if ( + metadata.st_uid != os.geteuid() + or metadata.st_gid != os.getegid() + or stat.S_IMODE(metadata.st_mode) != 0o700 + ): + _fail("candidate-directory-invalid") + release_payload = _canonical(binding) + b"\n" + ack = { + "schemaVersion": SCHEMA_VERSION, + "protocol": RELEASE_ACK_PROTOCOL, + "transactionId": binding["transactionId"], + "nonce": binding["nonce"], + "releaseSha256": hashlib.sha256(release_payload).hexdigest(), + "start": binding["start"], + } + payload = _canonical(ack) + b"\n" + for name in (RELEASE_ACK_NAME, RELEASE_ACK_PENDING_NAME): + existing = _read_at( + directory_fd, + name, + uid=os.geteuid(), + gid=os.getegid(), + mode=0o600, + missing=True, + ) + if existing is not None: + if existing != payload: + _fail("release-ack-conflict") + return str(binding["nonce"]) + temporary = f".{RELEASE_ACK_NAME}.{os.getpid()}.{secrets.token_hex(8)}" + fd = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=directory_fd, + ) + try: + view = memoryview(payload) + while view: + written = os.write(fd, view) + if written <= 0: + _fail("release-ack-write-failed") + view = view[written:] + os.fchmod(fd, 0o600) + os.fsync(fd) + finally: + os.close(fd) + os.replace( + temporary, + RELEASE_ACK_PENDING_NAME, + src_dir_fd=directory_fd, + dst_dir_fd=directory_fd, + ) + os.fsync(directory_fd) + return str(binding["nonce"]) + except OSError: + _fail("release-ack-write-failed") + finally: + os.close(directory_fd) + + def _run(action: str) -> str: directory_fd = _active_directory() if not _active_exists(directory_fd): if directory_fd is not None: os.close(directory_fd) + if action == "acknowledge": + _fail("activation-release-missing") return "inactive" assert directory_fd is not None try: @@ -628,6 +700,8 @@ def _run(action: str) -> str: ) if released is not None: _verify_release_candidate(released) + if action == "acknowledge": + return _prepare_release_ack(released) return "released" retry = _read_binding(directory_fd, RETRY_NAME, RETRY_PROTOCOL, RETRY_KEYS) if retry is not None: @@ -653,12 +727,20 @@ def _run(action: str) -> str: def main(argv: list[str] | None = None) -> int: arguments = sys.argv[1:] if argv is None else argv - if arguments not in (["admit"], ["checkpoint"], ["restart"], ["resume"]): + if arguments not in ( + ["admit"], + ["checkpoint"], + ["restart"], + ["resume"], + ["acknowledge"], + ): print("runtime-state-mutation-startup-gate: invalid action", file=sys.stderr) return 64 try: state = _run(arguments[0]) print(state) + if arguments[0] == "acknowledge": + return 0 return { "inactive": 0, "released": 0, diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 735a909c92b..dab7bc25f7f 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { execFileSync } from "node:child_process"; +import { execFileSync, spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -50,19 +52,36 @@ function ownerThatStopsAfterPrepare(runtime: ReturnType) { }; } +async function waitForPath(filePath: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + fs.accessSync(filePath); + return; + } catch {} + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`timed out waiting for ${path.basename(filePath)}`); +} + afterEach(() => { cleanupDockerStateMutationRoots(); }); describe("Docker runtime-provider state mutation surface", () => { - it("keeps the detached activation broker alive through retained-fence recovery (#10155)", () => { + it("derives the activation broker deadline from retry and readiness windows (#10155)", () => { const controller = path.join( import.meta.dirname, "../../../../scripts/runtime-state-mutation-control.py", ); + const controllerSource = fs.readFileSync(controller, "utf8"); + const processStateWindow = /^PROCESS_STATE_SECONDS = ([1-9][0-9]*(?:\.[0-9]+)?)$/mu.exec( + controllerSource, + ); const activationWindow = /^ACTIVATION_SECONDS = ([1-9][0-9]*(?:\.[0-9]+)?)$/mu.exec( - fs.readFileSync(controller, "utf8"), + controllerSource, ); + const processStateWindowSeconds = Number(processStateWindow?.[1] ?? Number.NaN); const activationWindowSeconds = Number(activationWindow?.[1] ?? Number.NaN); const brokerTimeouts = JSON.parse( /^TIMEOUTS = (\{[^\n]+\})$/mu.exec( @@ -70,9 +89,11 @@ describe("Docker runtime-provider state mutation surface", () => { )?.[1] ?? "null", ) as Record | null; + expect(processStateWindowSeconds).toBeGreaterThan(0); expect(activationWindowSeconds).toBeGreaterThan(0); - expect(brokerTimeouts?.activate).toEqual(expect.any(Number)); - expect(brokerTimeouts?.activate as number).toBeGreaterThan(activationWindowSeconds * 3); + expect(brokerTimeouts?.activate).toBe( + processStateWindowSeconds + 2 * activationWindowSeconds + 30, + ); expect(brokerTimeouts?.release).toBe(300); }); @@ -151,7 +172,74 @@ print(json.dumps(timeouts)) timeout: 5_000, }), ), - ).toEqual([480, 255]); + ).toEqual([335, 110]); + }); + + it("publishes a bounded activation timeout after the helper deadline (#10155)", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-state-mutation-broker-")); + const helper = path.join(root, "slow-helper.py"); + const transaction = randomBytes(32).toString("hex"); + const session = path.join(root, transaction); + const uid = process.getuid?.() ?? 0; + const gid = process.getgid?.() ?? 0; + const brokerSource = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.replace( + 'ROOT = "/run/nemoclaw/runtime-state-mutation"', + `ROOT = ${JSON.stringify(root)}`, + ) + .replace(/"activate": [0-9.]+/u, '"activate": 0.05') + .replaceAll( + "metadata.st_uid != 0 or metadata.st_gid != 0", + `metadata.st_uid != ${uid} or metadata.st_gid != ${gid}`, + ) + .replaceAll( + "before.st_uid != 0 or before.st_gid != 0", + `before.st_uid != ${uid} or before.st_gid != ${gid}`, + ); + fs.writeFileSync(helper, "import time\ntime.sleep(1)\n", { mode: 0o500 }); + const broker = spawn("python3", ["-I", "-c", brokerSource, helper, transaction], { + stdio: ["ignore", "ignore", "pipe"], + }); + const brokerExit = new Promise((resolve, reject) => { + broker.once("exit", () => resolve()); + broker.once("error", reject); + }); + let brokerStderr = ""; + broker.stderr.setEncoding("utf8"); + broker.stderr.on("data", (chunk: string) => { + brokerStderr += chunk; + }); + + try { + await waitForPath(path.join(session, "ready"), 2_000); + const request = Buffer.from( + `${JSON.stringify({ action: "activate", transactionId: transaction })}\n`, + "utf8", + ); + const identity = createHash("sha256").update(request).digest("hex"); + const responsePath = path.join(session, `${identity}.response`); + const startedAt = Date.now(); + fs.writeFileSync(path.join(session, `${identity}.activate.incoming`), request, { + mode: 0o600, + }); + await waitForPath(responsePath, 1_000); + + const response = JSON.parse(fs.readFileSync(responsePath, "utf8")); + expect(response).toEqual({ + schemaVersion: 1, + action: "activate", + identity, + status: 1, + stdout: "", + stderr: + '{"schemaVersion":1,"action":"activate","status":"failed","code":"helper-timeout"}\n', + }); + expect(Date.now() - startedAt).toBeLessThan(500); + } finally { + broker.kill("SIGTERM"); + await brokerExit; + fs.rmSync(root, { force: true, recursive: true }); + } + expect(brokerStderr).toBe(""); }); it("uses one harness-owned absolute Docker executable", () => { @@ -409,8 +497,8 @@ describe("Docker state mutation owner", () => { ["acquire", 30_000], ["assert", 30_000], ["rollback", 15 * 60_000], - ["activate", 8 * 60_000], - ["activate", 8 * 60_000], + ["activate", 335_000], + ["activate", 335_000], ["release", 5 * 60_000], ]); const acquireRequest = JSON.parse(helperCalls[0]?.[3]?.toString("utf8") ?? "null"); diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index c50c4c130fa..5c308b8a4b6 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -54,16 +54,21 @@ const SUPPORTED_STATE_ROOT = "/sandbox/.hermes"; const HELPER_PYTHON_PATH = "/opt/hermes/.venv/bin/python3"; const HELPER_PATH = "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py"; const HELPER_FAST_TIMEOUT_MS = 30_000; -// Retained activation recovery can consume three consecutive 150-second -// controller windows: retry acknowledgement, startup checkpoint, and live -// service verification. Keep the broker outside that bound and below the -// existing 15-minute Shields state-mutation guard. -const HELPER_ACTIVATION_TIMEOUT_MS = 8 * 60_000; +const HELPER_ACTIVATION_RETRY_ACK_TIMEOUT_MS = 5_000; +const HELPER_ACTIVATION_CHECKPOINT_TIMEOUT_MS = 150_000; +const HELPER_ACTIVATION_HEALTH_TIMEOUT_MS = 150_000; +const HELPER_ACTIVATION_COMPLETION_ALLOWANCE_MS = 30_000; +const HELPER_ACTIVATION_TIMEOUT_MS = + HELPER_ACTIVATION_RETRY_ACK_TIMEOUT_MS + + HELPER_ACTIVATION_CHECKPOINT_TIMEOUT_MS + + HELPER_ACTIVATION_HEALTH_TIMEOUT_MS + + HELPER_ACTIVATION_COMPLETION_ALLOWANCE_MS; const HELPER_RELEASE_TIMEOUT_MS = 5 * 60_000; export const DOCKER_STATE_MUTATION_GUARD_TIMEOUT_MS = 15 * 60_000; const INSPECT_TIMEOUT_MS = 15_000; const SUPERVISOR_SIGNAL_TIMEOUT_MS = 15_000; const HELPER_TRANSPORT_COMMAND_TIMEOUT_MS = 15_000; +const HELPER_TRANSPORT_RESPONSE_ALLOWANCE_MS = 30_000; const HELPER_TRANSPORT_POLL_MS = 250; const HELPER_TRANSPORT_ROOT = "/run/nemoclaw/runtime-state-mutation"; const MAX_HELPER_TRANSPORT_BYTES = 128 * 1024; @@ -1401,8 +1406,9 @@ function writePrivateTransportFile(filePath: string, value: Buffer): void { function copyHelperTransportFile( capture: HelperTransportCapture, command: PersistedEngineLifecycleExactCommand, + timeoutMs = HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, ): ContainerEngineCommandResult { - return capture(command, HELPER_TRANSPORT_COMMAND_TIMEOUT_MS); + return capture(command, timeoutMs); } function readHelperTransportFile( @@ -1416,10 +1422,13 @@ function readHelperTransportFile( const destination = path.join(temporary, "response"); const deadline = Date.now() + timeoutMs; while (true) { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) fail("root helper transport response did not arrive"); fs.rmSync(destination, { force: true }); const result = copyHelperTransportFile( capture, helperTransportCopyFromCommand(runtimeId, containerPath, destination), + Math.min(HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, remainingMs), ); if (!result.error && result.status === 0 && result.stderr.length === 0) { const value = fs.readFileSync(destination); @@ -1428,6 +1437,9 @@ function readHelperTransportFile( } return value; } + if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT") { + fail("root helper transport response copy timed out"); + } if (Date.now() >= deadline) fail("root helper transport response did not arrive"); Atomics.wait(helperTransportPoll, 0, 0, HELPER_TRANSPORT_POLL_MS); } @@ -1611,7 +1623,7 @@ function invokeHelperTransport( options.runtimeId, `${sessionPath}/${identity}.response`, options.hostTransportRoot, - helperTimeoutMs(action) + HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, + helperTimeoutMs(action) + HELPER_TRANSPORT_RESPONSE_ALLOWANCE_MS, ); const parsed = parseHelperTransportResult(response, action, identity); const acknowledgement = path.join(temporary, "ack"); diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index 72656db10fc..fd4a7e0089e 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -342,7 +342,7 @@ function createContainerStateMutationHarness( action === "acquire" || action === "assert" ? 30_000 : action === "activate" - ? 8 * 60_000 + ? 5_000 + 150_000 + 150_000 + 30_000 : action === "release" ? 5 * 60_000 : 15 * 60_000; diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 1e0d0bdef54..431c38f06f1 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -840,6 +840,7 @@ def resume_reference(reference): return state_process(by_release_pid[reference.pid]) control._resume_reference = resume_reference control._prove_released_activation = lambda *_args: release_events.append(["health"]) +control._wait_for_startup_release_ack = lambda *_args: None control._verify_activation_checkpoint = lambda *_args: release_events.append(["verify-checkpoint"]) control._publish_activation_release = lambda *_args: release_events.append(["release-receipt"]) terminated_release_pids = set() diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index 56c136a01ee..49d465bb802 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -416,12 +416,24 @@ describe("Hermes runtime state mutation publisher", () => { expect(start).toContain("trap nemoclaw_runtime_state_mutation_retry_exec USR2"); expect(start).toContain("exec /usr/local/bin/nemoclaw-start"); expect(start).toContain("nemoclaw_runtime_state_mutation_gate resume"); + expect(start).toContain("nemoclaw_runtime_state_mutation_acknowledge_release"); + expect(start).toMatch( + /nemoclaw_runtime_state_mutation_gate resume; then\n\s+if nemoclaw_runtime_state_mutation_acknowledge_release; then\n\s+return 0/u, + ); + expect(start).toContain('NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV="/usr/bin/mv"'); + expect(start).toContain( + '"$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV" -f -- "$pending" "$final"', + ); const startupGate = fs.readFileSync(STARTUP_GATE, "utf8"); expect(startupGate).toContain('DURABLE_DIRECTORY = "/var/lib/nemoclaw/runtime-state-mutation"'); expect(startupGate).toContain('"permitted": 10'); expect(startupGate).toContain('"activation-ready": 11'); expect(startupGate).toContain('"retry": 12'); + expect(startupGate).toContain( + 'RELEASE_ACK_PROTOCOL = "nemoclaw-runtime-state-mutation-release-ack-v1"', + ); + expect(startupGate).toContain('["acknowledge"]'); expect(fs.readFileSync(PUBLISHER, "utf8")).toContain( 'PYTHON_PATH = "/opt/hermes/.venv/bin/python3"', ); diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts new file mode 100644 index 00000000000..ec19b281a94 --- /dev/null +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const CONTROLLER = path.join( + import.meta.dirname, + "../../scripts/runtime-state-mutation-control.py", +); + +const HARNESS = String.raw` +import importlib.util +import json +import os +import sys +import tempfile + +spec = importlib.util.spec_from_file_location("runtime_state_control", sys.argv[1]) +control = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = control +spec.loader.exec_module(control) + +control.ROOT_UID = os.geteuid() +control.ROOT_GID = os.getegid() +control._sandbox_account = lambda: (os.geteuid(), os.getegid()) + +start = control.ProcessReference( + 10, + "101", + 1, + (os.geteuid(),) * 4, + "a" * 64, + 12, + 13, +) +fence = control.FenceProof(start, start, (os.geteuid(),)) +marker = {"transactionId": "c" * 64, "nonce": "b" * 64} +release_payload = b'{"release":"exact"}\n' + +def code(operation): + try: + operation() + return "ok" + except control.ControlError as error: + return error.code + +def write_at(directory_fd, name, payload): + fd = os.open( + name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + dir_fd=directory_fd, + ) + try: + os.write(fd, payload) + finally: + os.close(fd) + +results = {} +with tempfile.TemporaryDirectory() as root: + root = os.path.realpath(root) + os.chmod(root, 0o755) + control.STARTUP_HANDOFF_DIRECTORY = os.path.join(root, "handoff") + opened = control._open_startup_candidate_directory(marker, create=True) + assert opened is not None + root_fd, directory_fd = opened + try: + expected = control._startup_release_ack_payload( + marker, + fence, + release_payload, + ) + payload = control._canonical_protocol_payload(expected) + write_at( + directory_fd, + control.STARTUP_RELEASE_ACK_PENDING_NAME, + payload, + ) + results["pendingIgnored"] = ( + control._read_startup_release_ack(marker, fence, release_payload) + is None + ) + os.rename( + control.STARTUP_RELEASE_ACK_PENDING_NAME, + control.STARTUP_RELEASE_ACK_NAME, + src_dir_fd=directory_fd, + dst_dir_fd=directory_fd, + ) + results["committed"] = code( + lambda: control._read_startup_release_ack( + marker, + fence, + release_payload, + ) + ) + os.unlink(control.STARTUP_RELEASE_ACK_NAME, dir_fd=directory_fd) + wrong = {**expected, "releaseSha256": "0" * 64} + write_at( + directory_fd, + control.STARTUP_RELEASE_ACK_NAME, + control._canonical_protocol_payload(wrong), + ) + results["wrongRelease"] = code( + lambda: control._read_startup_release_ack( + marker, + fence, + release_payload, + ) + ) + write_at( + directory_fd, + control.STARTUP_RELEASE_ACK_PENDING_NAME, + payload, + ) + finally: + os.close(directory_fd) + os.close(root_fd) + + control._cleanup_startup_candidate_directory(marker) + results["cleaned"] = not os.path.exists( + control._startup_candidate_directory(marker) + ) + +print(json.dumps(results, sort_keys=True)) +`; + +describe("runtime state mutation release acknowledgement", () => { + it("accepts only a committed acknowledgement for the exact activation release (#10155)", () => { + const result = spawnSync("python3", ["-I", "-c", HARNESS, CONTROLLER], { + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + cleaned: true, + committed: "ok", + pendingIgnored: true, + wrongRelease: "activation-release-ack-invalid", + }); + }); +}); diff --git a/test/state/runtime-state-mutation-startup-gate.test.ts b/test/state/runtime-state-mutation-startup-gate.test.ts index ffaf4a0291d..81c46d97521 100644 --- a/test/state/runtime-state-mutation-startup-gate.test.ts +++ b/test/state/runtime-state-mutation-startup-gate.test.ts @@ -139,8 +139,23 @@ with tempfile.TemporaryDirectory() as root: "start": start, "candidateDirectory": candidate_directory, } - write(os.path.join(durable, gate.RELEASE_NAME), release, 0o444) + release_path = os.path.join(durable, gate.RELEASE_NAME) + write(release_path, release, 0o444) results["released"] = gate._run("admit") + results["release_ack_nonce"] = gate._run("acknowledge") + release_ack_pending = os.path.join( + candidate_directory, gate.RELEASE_ACK_PENDING_NAME + ) + with open(release_ack_pending, "rb") as stream: + results["release_ack"] = json.load(stream) + os.replace( + release_ack_pending, + os.path.join(candidate_directory, gate.RELEASE_ACK_NAME), + ) + results["release_ack_committed"] = gate._run("acknowledge") + gate._capture_parent = lambda: {**start, "pid": 42} + results["foreign_parent_ack"] = code(lambda: gate._run("acknowledge")) + gate._capture_parent = lambda: start with open(candidate_path, "ab") as stream: stream.write(b"tamper") results["tampered_release"] = code(lambda: gate._run("admit")) @@ -197,6 +212,9 @@ describe("runtime state mutation startup gate", () => { retry_wait: "retry-wait", retry_wrong_transaction: "retry-permit-mismatch", released: "released", + release_ack_nonce: "b".repeat(64), + release_ack_committed: "b".repeat(64), + foreign_parent_ack: "gate-start-mismatch", tampered_release: "release-candidate-mismatch", symlink_directory: "unsafe-directory", invalid_present_directory: "gate-directory-invalid", @@ -216,6 +234,14 @@ describe("runtime state mutation startup gate", () => { retrySha256: expect.stringMatching(/^[0-9a-f]{64}$/u), start: expectedStart, }, + release_ack: { + schemaVersion: 1, + protocol: "nemoclaw-runtime-state-mutation-release-ack-v1", + transactionId: "c".repeat(64), + nonce: "b".repeat(64), + releaseSha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + start: expectedStart, + }, }); }); }); From 4c9fd8b0974395a33d54ad0402b41f9f1ba8d47e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:00:15 -0700 Subject: [PATCH 06/44] fix(onboard): cover activation replay deadline Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- scripts/runtime-state-mutation-startup-gate.py | 1 + .../runtime-provider/docker-state-mutation.test.ts | 8 ++++---- src/lib/onboard/runtime-provider/docker-state-mutation.ts | 2 ++ test/helpers/docker-state-mutation-harness.ts | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index 6431c30bd73..ec448219faa 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -616,6 +616,7 @@ def _publish_retry_ack(binding: dict[str, object], retry_payload: bytes) -> None _fail("retry-ack-write-failed") finally: os.close(directory_fd) + return def _prepare_release_ack(binding: dict[str, object]) -> str: diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index dab7bc25f7f..932bc576a2c 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -92,7 +92,7 @@ describe("Docker runtime-provider state mutation surface", () => { expect(processStateWindowSeconds).toBeGreaterThan(0); expect(activationWindowSeconds).toBeGreaterThan(0); expect(brokerTimeouts?.activate).toBe( - processStateWindowSeconds + 2 * activationWindowSeconds + 30, + processStateWindowSeconds + 3 * activationWindowSeconds + 30, ); expect(brokerTimeouts?.release).toBe(300); }); @@ -172,7 +172,7 @@ print(json.dumps(timeouts)) timeout: 5_000, }), ), - ).toEqual([335, 110]); + ).toEqual([485, 260]); }); it("publishes a bounded activation timeout after the helper deadline (#10155)", async () => { @@ -497,8 +497,8 @@ describe("Docker state mutation owner", () => { ["acquire", 30_000], ["assert", 30_000], ["rollback", 15 * 60_000], - ["activate", 335_000], - ["activate", 335_000], + ["activate", 485_000], + ["activate", 485_000], ["release", 5 * 60_000], ]); const acquireRequest = JSON.parse(helperCalls[0]?.[3]?.toString("utf8") ?? "null"); diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index 5c308b8a4b6..6334c1a10b0 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -57,11 +57,13 @@ const HELPER_FAST_TIMEOUT_MS = 30_000; const HELPER_ACTIVATION_RETRY_ACK_TIMEOUT_MS = 5_000; const HELPER_ACTIVATION_CHECKPOINT_TIMEOUT_MS = 150_000; const HELPER_ACTIVATION_HEALTH_TIMEOUT_MS = 150_000; +const HELPER_ACTIVATION_REPLAY_TIMEOUT_MS = 150_000; const HELPER_ACTIVATION_COMPLETION_ALLOWANCE_MS = 30_000; const HELPER_ACTIVATION_TIMEOUT_MS = HELPER_ACTIVATION_RETRY_ACK_TIMEOUT_MS + HELPER_ACTIVATION_CHECKPOINT_TIMEOUT_MS + HELPER_ACTIVATION_HEALTH_TIMEOUT_MS + + HELPER_ACTIVATION_REPLAY_TIMEOUT_MS + HELPER_ACTIVATION_COMPLETION_ALLOWANCE_MS; const HELPER_RELEASE_TIMEOUT_MS = 5 * 60_000; export const DOCKER_STATE_MUTATION_GUARD_TIMEOUT_MS = 15 * 60_000; diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index fd4a7e0089e..57f9d201898 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -342,7 +342,7 @@ function createContainerStateMutationHarness( action === "acquire" || action === "assert" ? 30_000 : action === "activate" - ? 5_000 + 150_000 + 150_000 + 30_000 + ? 5_000 + 150_000 + 150_000 + 150_000 + 30_000 : action === "release" ? 5 * 60_000 : 15 * 60_000; From d22dff5fc46789d1b7bf611eee05ce18d96cc735 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 25 Aug 2026 15:09:43 -0700 Subject: [PATCH 07/44] fix(runtime): cover retained activation deadline Signed-off-by: Charan Jagwani --- test/helpers/docker-state-mutation-harness.ts | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index 57f9d201898..cc50b6819b7 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -346,6 +346,7 @@ function createContainerStateMutationHarness( : action === "release" ? 5 * 60_000 : 15 * 60_000; + const helperDeadline = Date.now() + helperTimeout; let helperResult = capture( "docker", [ @@ -356,23 +357,37 @@ function createContainerStateMutationHarness( "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", action, ], - helperTimeout, + Math.max(1, helperDeadline - Date.now()), request, ); if (helperResult.status !== null && helperResult.status < 0) { - helperResult = capture( - "docker", - [ - "container", - "exec", - "--nemoclaw-broker", - DOCKER_STATE_MUTATION_RUNTIME_ID, - "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", - action, - ], - helperTimeout, - request, - ); + const remainingTimeout = helperDeadline - Date.now(); + if (remainingTimeout <= 0) { + helperResult = { + status: 1, + stdout: "", + stderr: `${JSON.stringify({ + schemaVersion: 1, + action, + status: "failed", + code: "helper-timeout", + })}\n`, + }; + } else { + helperResult = capture( + "docker", + [ + "container", + "exec", + "--nemoclaw-broker", + DOCKER_STATE_MUTATION_RUNTIME_ID, + "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", + action, + ], + remainingTimeout, + request, + ); + } } transportFiles.set( `${path.posix.dirname(containerPath)}/${identity}.response`, From 079b9b14782df8b9d9ebef28062ef73c59a9d5c9 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Tue, 25 Aug 2026 15:27:52 -0700 Subject: [PATCH 08/44] fix(runtime): use full response allowance Signed-off-by: Charan Jagwani --- .../docker-state-mutation.test.ts | 15 +++++++++++++++ .../runtime-provider/docker-state-mutation.ts | 11 ++++++++--- test/helpers/docker-state-mutation-harness.ts | 13 +++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 932bc576a2c..c697c7a2af9 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -242,6 +242,21 @@ print(json.dumps(timeouts)) expect(brokerStderr).toBe(""); }); + it("uses the remaining response allowance after one Docker copy timeout (#10155)", () => { + const runtime = harness({ timeoutResponseCopyOnce: true }); + const surface = createDockerStateMutationSurface({ + capture: runtime.capture, + resolveStateDir: () => runtime.root, + }); + + expect(() => surface.acquire({ ...runtime.context, plan: plan() })).not.toThrow(); + const responseCopies = runtime.capture.mock.calls.filter(([, args]) => + args.some((value) => value.endsWith(".response")), + ); + expect(responseCopies).toHaveLength(2); + expect(responseCopies.map(([, , timeout]) => timeout)).toEqual([15_000, 15_000]); + }); + it("uses one harness-owned absolute Docker executable", () => { const runtime = harness(); runtime.authority.engine.capture(["version"]); diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index 6334c1a10b0..379622a3355 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -1439,10 +1439,15 @@ function readHelperTransportFile( } return value; } - if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT") { - fail("root helper transport response copy timed out"); + const copyTimedOut = + (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT"; + if (Date.now() >= deadline) { + fail( + copyTimedOut + ? "root helper transport response copy timed out" + : "root helper transport response did not arrive", + ); } - if (Date.now() >= deadline) fail("root helper transport response did not arrive"); Atomics.wait(helperTransportPoll, 0, 0, HELPER_TRANSPORT_POLL_MS); } }); diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index cc50b6819b7..f082d8ab024 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -150,6 +150,7 @@ export interface DockerStateMutationHarnessOptions { readonly loseReleaseResponseOnce?: boolean; readonly signalHelperOnce?: boolean; readonly stateMountType?: "bind" | "volume"; + readonly timeoutResponseCopyOnce?: boolean; } export interface DockerStateMutationHarnessState { @@ -197,6 +198,7 @@ function createContainerStateMutationHarness( let resumeFailuresRemaining = options.failResumeOnce ? 1 : 0; let lostReleaseResponsesRemaining = options.loseReleaseResponseOnce ? 1 : 0; let signalledHelpersRemaining = options.signalHelperOnce ? 1 : 0; + let responseCopyTimeoutsRemaining = options.timeoutResponseCopyOnce ? 1 : 0; let marker: Record | null = null; let releasedMarker: Record | null = null; let deferredAcquireRequest: string | null = null; @@ -434,6 +436,17 @@ function createContainerStateMutationHarness( } if (source.startsWith(containerPrefix)) { const containerPath = source.slice(containerPrefix.length); + if (containerPath.endsWith(".response") && responseCopyTimeoutsRemaining > 0) { + responseCopyTimeoutsRemaining -= 1; + return { + error: Object.assign(new Error("transport response copy timed out"), { + code: "ETIMEDOUT", + }), + status: 1, + stdout: "", + stderr: "", + }; + } const payload = transportFiles.get(containerPath); if (!payload) return { status: 1, stdout: "", stderr: "transport file unavailable" }; fs.writeFileSync(destination, payload, { mode: 0o600 }); From 60c2567be4ddae75158ec749866f194d1e37bd05 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 26 Aug 2026 01:51:14 -0700 Subject: [PATCH 09/44] fix(runtime): make release acknowledgement failure explicit Raise the fixed GateError directly when release acknowledgement publication fails. Cover the redacted failure result. Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- scripts/runtime-state-mutation-startup-gate.py | 2 +- test/state/runtime-state-mutation-startup-gate.test.ts | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index ec448219faa..110f68f11c2 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -681,7 +681,7 @@ def _prepare_release_ack(binding: dict[str, object]) -> str: os.fsync(directory_fd) return str(binding["nonce"]) except OSError: - _fail("release-ack-write-failed") + raise GateError("release-ack-write-failed") from None finally: os.close(directory_fd) diff --git a/test/state/runtime-state-mutation-startup-gate.test.ts b/test/state/runtime-state-mutation-startup-gate.test.ts index 81c46d97521..263a2de6f3e 100644 --- a/test/state/runtime-state-mutation-startup-gate.test.ts +++ b/test/state/runtime-state-mutation-startup-gate.test.ts @@ -142,6 +142,14 @@ with tempfile.TemporaryDirectory() as root: release_path = os.path.join(durable, gate.RELEASE_NAME) write(release_path, release, 0o444) results["released"] = gate._run("admit") + original_replace = gate.os.replace + + def fail_release_ack_replace(*_args, **_kwargs): + raise OSError("sensitive fixture detail") + + gate.os.replace = fail_release_ack_replace + results["release_ack_write_failure"] = code(lambda: gate._run("acknowledge")) + gate.os.replace = original_replace results["release_ack_nonce"] = gate._run("acknowledge") release_ack_pending = os.path.join( candidate_directory, gate.RELEASE_ACK_PENDING_NAME @@ -212,6 +220,7 @@ describe("runtime state mutation startup gate", () => { retry_wait: "retry-wait", retry_wrong_transaction: "retry-permit-mismatch", released: "released", + release_ack_write_failure: "release-ack-write-failed", release_ack_nonce: "b".repeat(64), release_ack_committed: "b".repeat(64), foreign_parent_ack: "gate-start-mismatch", From f9e7e51cadf32fef4b380c6afa380ec1f4b06b91 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 26 Aug 2026 06:27:04 -0700 Subject: [PATCH 10/44] fix(runtime): resolve release acknowledgement feedback Use the common fixed-failure path for release acknowledgement writes. Strengthen the broker deadline and acknowledgement rejection evidence requested in review. Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../runtime-state-mutation-startup-gate.py | 2 +- .../docker-state-mutation.test.ts | 23 +------ ...runtime-state-mutation-release-ack.test.ts | 64 +++++++++++++++---- 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index 110f68f11c2..ec448219faa 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -681,7 +681,7 @@ def _prepare_release_ack(binding: dict[str, object]) -> str: os.fsync(directory_fd) return str(binding["nonce"]) except OSError: - raise GateError("release-ack-write-failed") from None + _fail("release-ack-write-failed") finally: os.close(directory_fd) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index c697c7a2af9..b9aaf21f9da 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -69,31 +69,14 @@ afterEach(() => { }); describe("Docker runtime-provider state mutation surface", () => { - it("derives the activation broker deadline from retry and readiness windows (#10155)", () => { - const controller = path.join( - import.meta.dirname, - "../../../../scripts/runtime-state-mutation-control.py", - ); - const controllerSource = fs.readFileSync(controller, "utf8"); - const processStateWindow = /^PROCESS_STATE_SECONDS = ([1-9][0-9]*(?:\.[0-9]+)?)$/mu.exec( - controllerSource, - ); - const activationWindow = /^ACTIVATION_SECONDS = ([1-9][0-9]*(?:\.[0-9]+)?)$/mu.exec( - controllerSource, - ); - const processStateWindowSeconds = Number(processStateWindow?.[1] ?? Number.NaN); - const activationWindowSeconds = Number(activationWindow?.[1] ?? Number.NaN); + it("uses the bounded activation deadline at the broker boundary (#10155)", () => { const brokerTimeouts = JSON.parse( /^TIMEOUTS = (\{[^\n]+\})$/mu.exec( DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE, )?.[1] ?? "null", ) as Record | null; - expect(processStateWindowSeconds).toBeGreaterThan(0); - expect(activationWindowSeconds).toBeGreaterThan(0); - expect(brokerTimeouts?.activate).toBe( - processStateWindowSeconds + 3 * activationWindowSeconds + 30, - ); + expect(brokerTimeouts?.activate).toBe(485); expect(brokerTimeouts?.release).toBe(300); }); @@ -234,12 +217,12 @@ print(json.dumps(timeouts)) '{"schemaVersion":1,"action":"activate","status":"failed","code":"helper-timeout"}\n', }); expect(Date.now() - startedAt).toBeLessThan(500); + expect(brokerStderr, brokerStderr).toBe(""); } finally { broker.kill("SIGTERM"); await brokerExit; fs.rmSync(root, { force: true, recursive: true }); } - expect(brokerStderr).toBe(""); }); it("uses the remaining response allowance after one Docker copy timeout (#10155)", () => { diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index ec19b281a94..8c9f705c6df 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -12,6 +12,7 @@ const CONTROLLER = path.join( ); const HARNESS = String.raw` +import hashlib import importlib.util import json import os @@ -59,6 +60,21 @@ def write_at(directory_fd, name, payload): finally: os.close(fd) +def record_rejection(directory_fd, name, payload): + write_at( + directory_fd, + control.STARTUP_RELEASE_ACK_NAME, + control._canonical_protocol_payload(payload), + ) + results[name] = code( + lambda: control._read_startup_release_ack( + marker, + fence, + release_payload, + ) + ) + os.unlink(control.STARTUP_RELEASE_ACK_NAME, dir_fd=directory_fd) + results = {} with tempfile.TemporaryDirectory() as root: root = os.path.realpath(root) @@ -68,7 +84,23 @@ with tempfile.TemporaryDirectory() as root: assert opened is not None root_fd, directory_fd = opened try: - expected = control._startup_release_ack_payload( + expected = { + "schemaVersion": 1, + "protocol": "nemoclaw-runtime-state-mutation-release-ack-v1", + "transactionId": "c" * 64, + "nonce": "b" * 64, + "releaseSha256": hashlib.sha256(release_payload).hexdigest(), + "start": { + "pid": 10, + "startIdentity": "101", + "parentPid": 1, + "uids": [os.geteuid()] * 4, + "commandSha256": "a" * 64, + "procDevice": "12", + "procInode": "13", + }, + } + assert expected == control._startup_release_ack_payload( marker, fence, release_payload, @@ -97,18 +129,25 @@ with tempfile.TemporaryDirectory() as root: ) ) os.unlink(control.STARTUP_RELEASE_ACK_NAME, dir_fd=directory_fd) - wrong = {**expected, "releaseSha256": "0" * 64} - write_at( + record_rejection( directory_fd, - control.STARTUP_RELEASE_ACK_NAME, - control._canonical_protocol_payload(wrong), + "wrongRelease", + {**expected, "releaseSha256": "0" * 64}, ) - results["wrongRelease"] = code( - lambda: control._read_startup_release_ack( - marker, - fence, - release_payload, - ) + record_rejection( + directory_fd, + "wrongNonce", + {**expected, "nonce": "d" * 64}, + ) + record_rejection( + directory_fd, + "wrongTransaction", + {**expected, "transactionId": "d" * 64}, + ) + record_rejection( + directory_fd, + "wrongStart", + {**expected, "start": {**expected["start"], "pid": 11}}, ) write_at( directory_fd, @@ -137,7 +176,10 @@ describe("runtime state mutation release acknowledgement", () => { cleaned: true, committed: "ok", pendingIgnored: true, + wrongNonce: "activation-release-ack-invalid", wrongRelease: "activation-release-ack-invalid", + wrongStart: "activation-release-ack-invalid", + wrongTransaction: "activation-release-ack-invalid", }); }); }); From 83519cfd7810190a38724dc15432621f694cb0b9 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:06:23 -0700 Subject: [PATCH 11/44] fix(runtime): make acknowledgement failure return explicit Return through the fixed helper so static analysis proves that the branch cannot fall through. Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- scripts/runtime-state-mutation-startup-gate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index ec448219faa..dc3caceeb5c 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -681,7 +681,7 @@ def _prepare_release_ack(binding: dict[str, object]) -> str: os.fsync(directory_fd) return str(binding["nonce"]) except OSError: - _fail("release-ack-write-failed") + return _fail("release-ack-write-failed") finally: os.close(directory_fd) From 8ece9a0d32c29d7d68602caafd16fb60f0a0efb4 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 26 Aug 2026 08:10:24 -0700 Subject: [PATCH 12/44] test(runtime): diagnose repeated Shields transitions --- .../docker-state-mutation.test.ts | 41 +++++++++++++++++++ .../runtime-provider/docker-state-mutation.ts | 11 +++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index b9aaf21f9da..278441b2d58 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -528,6 +528,47 @@ describe("Docker state mutation owner", () => { expect(inspectFormats.every((format) => !format.includes("Config.Env"))).toBe(true); }); + it("fully releases transport before a back-to-back provider transition (#10155)", () => { + const runtime = harness(); + + const completeTransition = (completedLedgerSha256: string) => { + const fence = runtime.owner.acquire({ ...runtime.context, plan: plan() }); + runtime.owner.assertFenced(runtime.context, fence); + runtime.owner.publish(runtime.context, fence); + runtime.owner.assertFenced(runtime.context, fence); + const proof = runtime.owner.activate(runtime.context, fence); + runtime.owner.release(runtime.context, fence, proof, completedLedgerSha256); + return fence; + }; + + const first = completeTransition("e".repeat(64)); + expect(runtime.transportBrokerActive()).toBe(false); + expect(runtime.lifecycleStore.listUnfinished()).toEqual([]); + + const second = completeTransition("f".repeat(64)); + + expect(second.transactionId).not.toBe(first.transactionId); + expect(runtime.transportBrokerActive()).toBe(false); + expect(runtime.lifecycleStore.listUnfinished()).toEqual([]); + expect(runtime.helperActions).toEqual([ + "acquire", + "assert", + "publish", + "assert", + "activate", + "activate", + "release", + "acquire", + "assert", + "publish", + "assert", + "activate", + "activate", + "release", + ]); + expect(runtime.supervisorSignals).toEqual(["SIGSTOP", "SIGCONT", "SIGSTOP", "SIGCONT"]); + }); + it("publishes without retiring the host lifecycle fence", async () => { const runtime = harness(); const fence = await runtime.owner.acquire({ ...runtime.context, plan: plan() }); diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index 379622a3355..267aa385a9e 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -1419,13 +1419,14 @@ function readHelperTransportFile( containerPath: string, hostRoot: string, timeoutMs: number, + action: HelperAction | "startup", ): Buffer { return withHelperTransportHostDirectory(hostRoot, (temporary) => { const destination = path.join(temporary, "response"); const deadline = Date.now() + timeoutMs; while (true) { const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) fail("root helper transport response did not arrive"); + if (remainingMs <= 0) fail(`root helper ${action} transport response did not arrive`); fs.rmSync(destination, { force: true }); const result = copyHelperTransportFile( capture, @@ -1435,7 +1436,7 @@ function readHelperTransportFile( if (!result.error && result.status === 0 && result.stderr.length === 0) { const value = fs.readFileSync(destination); if (value.byteLength > MAX_HELPER_TRANSPORT_BYTES) { - fail("root helper transport response exceeds its byte bound"); + fail(`root helper ${action} transport response exceeds its byte bound`); } return value; } @@ -1444,8 +1445,8 @@ function readHelperTransportFile( if (Date.now() >= deadline) { fail( copyTimedOut - ? "root helper transport response copy timed out" - : "root helper transport response did not arrive", + ? `root helper ${action} transport response copy timed out` + : `root helper ${action} transport response did not arrive`, ); } Atomics.wait(helperTransportPoll, 0, 0, HELPER_TRANSPORT_POLL_MS); @@ -1500,6 +1501,7 @@ function ensureHelperTransportAuthorized( `${helperTransportSessionPath(transactionId)}/ready`, options.hostTransportRoot, HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, + "startup", ); } catch { fail("root helper transport did not become available"); @@ -1631,6 +1633,7 @@ function invokeHelperTransport( `${sessionPath}/${identity}.response`, options.hostTransportRoot, helperTimeoutMs(action) + HELPER_TRANSPORT_RESPONSE_ALLOWANCE_MS, + action, ); const parsed = parseHelperTransportResult(response, action, identity); const acknowledgement = path.join(temporary, "ack"); From f50ff0f1183f51209fe405737602f4574cb2c2d7 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:36:16 -0700 Subject: [PATCH 13/44] test(ci): refresh Hermes Shields validation Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> From f6a02e56ea6ddf96c6f3eb2fb7e94edb36646540 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 26 Aug 2026 12:26:26 -0700 Subject: [PATCH 14/44] fix(onboard): wait for verified create handoff Signed-off-by: Charan Jagwani --- src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts | 1 + src/lib/onboard/sandbox-gpu-create-run-attempt.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 26bf2518f23..ae42710aa7a 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -135,6 +135,7 @@ describe("created sandbox identity gate", () => { mocks.streamSandboxCreate.mockImplementation(async (_command, args, _env, options) => { events.push("create"); expect(options.onPoll).toBeUndefined(); + expect(options.waitForReadyTermination).toBe(true); expect(args.indexOf("--label")).toBeGreaterThan(0); expect(args.indexOf("--label")).toBeLessThan(args.indexOf("--")); nonce = createAttemptNonce(args); diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index fd2c788607a..0e10b8e6f9b 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -479,7 +479,7 @@ export function createSandboxGpuCreateAttemptRunner( }), failureCheck: runtimePatch.createFailureMessage, traceEvent: addTraceEvent, - waitForReadyTermination: deferRestartSafeCutover, + waitForReadyTermination: deferRestartSafeCutover || deferPostCreateEffects, initialPhase: compatibility && (input.prebuild.imageRef || state.compatibilityArgv) ? "create" From 77550ea3a5838425ff0fcc364fc8e8b369a37aa6 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 26 Aug 2026 14:13:09 -0700 Subject: [PATCH 15/44] fix(onboard): wait for exact sandbox publication Signed-off-by: Charan Jagwani --- .../sandbox-gpu-create-identity-gate.test.ts | 39 +++++++++++++++ .../onboard/sandbox-gpu-create-run-attempt.ts | 50 +++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index ae42710aa7a..36d5709f888 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -197,6 +197,45 @@ describe("created sandbox identity gate", () => { ); }); + it("waits for the exact created sandbox to appear through its owning gateway (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(deps.runOpenshell) + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: + "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", + }) + .mockReturnValue({ + status: 0, + stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", + stderr: "", + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); + + expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(1); + expect(deps.runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "get", "-g", "nemoclaw", "alpha"], + expect.objectContaining({ ignoreError: true, suppressOutput: true }), + ); + expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); + }); + it("rejects a same-name replacement before post-create effects (#9833)", async () => { let nonce = ""; const outputCanary = "replacement-output-must-not-be-reported"; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 0e10b8e6f9b..2639bc918b1 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -63,6 +63,7 @@ export type SandboxGpuCreateAttemptState = { // to live validation or the GPU proof. const REPLACEMENT_STABLE_READY_POLLS = 2; const SANDBOX_READY_PROBE_TIMEOUT_MS = 5_000; +const CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS = 1; const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/gu; const OPENSHELL_SANDBOX_NOT_READY = @@ -218,6 +219,53 @@ async function verifyCreatedSandboxBeforeEffects( }); } +function waitForCreatedOpenShellSandboxPublication( + sandboxId: string, + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, +): void { + const timeoutMs = Math.max(1, Math.round(input.sandboxReadyTimeoutSecs * 1_000)); + const deadlineMs = Date.now() + timeoutMs; + const maxPolls = + Math.ceil(timeoutMs / (CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS * 1_000)) + 1; + for (let poll = 0; poll < maxPolls; poll += 1) { + const remainingMs = Math.max(1, deadlineMs - Date.now()); + const result = deps.runOpenshell( + ["sandbox", "get", "-g", input.gatewayName, input.sandboxName], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(SANDBOX_READY_PROBE_TIMEOUT_MS, remainingMs), + killSignal: "SIGKILL", + }, + ); + if (result.status === 0 && !result.error) { + const publishedSandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); + if (!publishedSandboxId) { + throw new Error( + `OpenShell returned no exact durable ID for created sandbox '${input.sandboxName}'.`, + ); + } + if (publishedSandboxId !== sandboxId) { + throw new Error( + `Created sandbox '${input.sandboxName}' changed identity before policy verification.`, + ); + } + return; + } + if (poll + 1 >= maxPolls || Date.now() >= deadlineMs) break; + deps.sleep( + Math.min( + CREATED_SANDBOX_PUBLICATION_POLL_INTERVAL_SECONDS, + Math.max(0, (deadlineMs - Date.now()) / 1_000), + ), + ); + } + throw new Error( + `Created sandbox '${input.sandboxName}' did not become visible through its owning gateway before policy verification.`, + ); +} + function checkRecreatedSandboxReadyIdentity( sandboxName: string, expectedSandboxId: string, @@ -557,6 +605,7 @@ export function createSandboxGpuCreateAttemptRunner( { cause: error }, ); } + waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); await verifyCreatedSandboxBeforeEffects(sandboxId, route, input); createdSandboxVerified = true; if (deferPostCreateEffects) { @@ -682,6 +731,7 @@ export function createSandboxGpuCreateAttemptRunner( { cause: error }, ); } + waitForCreatedOpenShellSandboxPublication(sandboxId, input, deps); await verifyCreatedSandboxBeforeEffects(sandboxId, route, input); createdSandboxVerified = true; } From 24c8a0d88526ca1918c0ec7c29c5850736bed1e3 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 26 Aug 2026 14:38:10 -0700 Subject: [PATCH 16/44] test(onboard): cover sandbox publication rejection Signed-off-by: Charan Jagwani --- .../sandbox-gpu-create-identity-gate.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 3ac202f02cc..3def961cf9e 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -236,6 +236,70 @@ describe("created sandbox identity gate", () => { expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); }); + it("rejects a different owner-scoped sandbox identity before post-create effects (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(deps.runOpenshell).mockReturnValue({ + status: 0, + stdout: "Name: alpha\nId: replacement-sandbox-id\nState: Ready\n", + stderr: "", + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "changed identity before policy verification", + ); + + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + + it("stops when owner-scoped sandbox publication exceeds the deadline (#9833)", async () => { + let nonce = ""; + const input = noGpuInput(); + input.sandboxReadyTimeoutSecs = 0.001; + input.verifyCreatedSandboxBeforeEffects = vi.fn(); + input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); + const patch = createGpuPatchFixture(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { + nonce = createAttemptNonce(args); + return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; + }); + const deps = createGpuFlowDeps(); + vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => + sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), + ); + vi.mocked(deps.runOpenshell).mockReturnValue({ + status: 1, + stdout: "", + stderr: + "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( + "did not become visible through its owning gateway before policy verification", + ); + + expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); + expect(patch.exitOnPatchError).not.toHaveBeenCalled(); + expect(patch.ensureApplied).not.toHaveBeenCalled(); + expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); + }); + it("rejects a same-name replacement before post-create effects (#9833)", async () => { let nonce = ""; const outputCanary = "replacement-output-must-not-be-reported"; From 4ee50476d3ebe8a9f1c6321075dadc688a679ebc Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 26 Aug 2026 16:24:50 -0700 Subject: [PATCH 17/44] fix(hermes): preserve startup identity for release ack Signed-off-by: Charan Jagwani --- agents/hermes/start.sh | 31 ++++++-- .../docker-state-mutation.test.ts | 11 --- ...me-state-mutation-hermes-publisher.test.ts | 76 ++++++++++++++++--- 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index de6ef0ba95e..b14d109d188 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -31,12 +31,16 @@ readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON="/opt/hermes/.venv/bin/pyth readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER="/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py" readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV="/usr/bin/setpriv" readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV="/usr/bin/mv" +readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP="/usr/bin/mktemp" +readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM="/usr/bin/rm" readonly NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT="/run/nemoclaw/runtime-state-mutation-startup" if [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" ] \ || [ ! -f "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \ || [ -L "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \ || [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV" ] \ + || [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP" ] \ + || [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" ] \ || { [ "$EUID" -eq 0 ] && [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" ]; }; then printf '%s\n' '[SECURITY] Required runtime state mutation startup gate is unavailable.' >&2 exit 1 @@ -56,16 +60,33 @@ nemoclaw_runtime_state_mutation_gate() { } nemoclaw_runtime_state_mutation_acknowledge_release() { - local nonce pending final + local nonce nonce_file pending final + nonce_file="$("$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP" \ + /tmp/nemoclaw-runtime-state-mutation-ack.XXXXXXXXXX)" || return 1 + if [ ! -f "$nonce_file" ] || [ -L "$nonce_file" ]; then + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" + return 1 + fi if [ "$EUID" -eq 0 ]; then - nonce="$("$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" \ + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" \ --reuid=sandbox --regid=sandbox --init-groups -- \ "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge)" || return 1 + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >"$nonce_file" || { + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" + return 1 + } else - nonce="$("$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge)" || return 1 + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >"$nonce_file" || { + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" + return 1 + } + fi + if ! IFS= read -r nonce <"$nonce_file"; then + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" + return 1 fi + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" if [ "${#nonce}" -ne 64 ]; then return 1 fi diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 278441b2d58..7ee1c0a7e46 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -69,17 +69,6 @@ afterEach(() => { }); describe("Docker runtime-provider state mutation surface", () => { - it("uses the bounded activation deadline at the broker boundary (#10155)", () => { - const brokerTimeouts = JSON.parse( - /^TIMEOUTS = (\{[^\n]+\})$/mu.exec( - DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE, - )?.[1] ?? "null", - ) as Record | null; - - expect(brokerTimeouts?.activate).toBe(485); - expect(brokerTimeouts?.release).toBe(300); - }); - it("preserves safe broker diagnostics after request validation", () => { const definitionsEnd = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.indexOf( "\nhelper = sys.argv[1]\n", diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index 49d465bb802..6e14f9bb4cc 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -3,6 +3,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; @@ -328,6 +329,14 @@ function runHarness(): Record { return JSON.parse(result.stdout) as Record; } +function shellFunction(source: string, name: string, nextName: string): string { + const start = source.indexOf(`${name}() {`); + const end = source.indexOf(`\n${nextName}() {`, start); + expect(start, `Expected ${name} in agents/hermes/start.sh`).toBeGreaterThanOrEqual(0); + expect(end, `Expected ${nextName} after ${name} in agents/hermes/start.sh`).toBeGreaterThan(start); + return source.slice(start, end); +} + describe("Hermes runtime state mutation publisher", () => { it("publishes and rolls back only the exact installed full plan (#7744)", () => { const result = runHarness(); @@ -394,6 +403,61 @@ describe("Hermes runtime state mutation publisher", () => { ]); }); + it("publishes the release acknowledgement from the Hermes startup process (#10155)", () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-release-ack-")); + try { + const nonce = "a".repeat(64); + const gate = path.join(temporary, "gate.py"); + const waited = path.join(temporary, "controller-wait-finished"); + fs.writeFileSync( + gate, + `import json\nimport os\nimport sys\n\nexpected = int(os.environ["NEMOCLAW_TEST_EXPECTED_START_PID"])\nif os.getppid() != expected:\n raise SystemExit("gate-start-mismatch")\nnonce = os.environ["NEMOCLAW_TEST_NONCE"]\nroot = os.environ["NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT"]\ndirectory = os.path.join(root, nonce)\nos.mkdir(directory, 0o700)\npending = os.path.join(directory, ".release-ack.json.pending")\nwith open(pending, "x", encoding="utf-8") as stream:\n json.dump({"nonce": nonce}, stream, separators=(",", ":"))\nos.chmod(pending, 0o600)\nprint(nonce)\n`, + { mode: 0o700 }, + ); + const start = fs.readFileSync(START, "utf8"); + const acknowledge = shellFunction( + start, + "nemoclaw_runtime_state_mutation_acknowledge_release", + "nemoclaw_runtime_state_mutation_retry_exec", + ); + const controllerWait = + 'import os,sys,time\npath=sys.argv[1]\ndeadline=time.monotonic()+5\nwhile not os.path.isfile(path):\n if time.monotonic() >= deadline: raise SystemExit("release-ack-timeout")\n time.sleep(0.01)\nopen(sys.argv[2], "x").close()'; + const script = `${acknowledge} +export NEMOCLAW_TEST_EXPECTED_START_PID="$$" +"$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I -c "$NEMOCLAW_TEST_CONTROLLER_WAIT" \ + "$NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT/$NEMOCLAW_TEST_NONCE/release-ack.json" \ + "$NEMOCLAW_TEST_WAITED" & +controller_pid=$! +nemoclaw_runtime_state_mutation_acknowledge_release +wait "$controller_pid" +`; + const result = spawnSync("bash", ["-c", script], { + encoding: "utf8", + timeout: 10_000, + env: { + ...process.env, + NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON: "python3", + NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER: gate, + NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV: "setpriv", + NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV: "mv", + NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP: "mktemp", + NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM: "rm", + NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT: temporary, + NEMOCLAW_TEST_CONTROLLER_WAIT: controllerWait, + NEMOCLAW_TEST_NONCE: nonce, + NEMOCLAW_TEST_WAITED: waited, + }, + }); + expect(result.status, result.stderr).toBe(0); + expect( + fs.readFileSync(path.join(temporary, nonce, "release-ack.json"), "utf8"), + ).toBe(`{"nonce":"${nonce}"}`); + expect(fs.existsSync(waited)).toBe(true); + } finally { + fs.rmSync(temporary, { force: true, recursive: true }); + } + }); + it("publishes one exact image capability and checks the durable root gate before startup code", () => { expect(fs.readFileSync(CAPABILITY, "utf8")).toBe( '{"schemaVersion":1,"protocol":"nemoclaw-runtime-state-mutation-publisher-v1","agent":"hermes","providerId":"docker","stateRoot":"/sandbox/.hermes","planSchemaVersion":2,"entrypoint":"/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py"}\n', @@ -416,24 +480,12 @@ describe("Hermes runtime state mutation publisher", () => { expect(start).toContain("trap nemoclaw_runtime_state_mutation_retry_exec USR2"); expect(start).toContain("exec /usr/local/bin/nemoclaw-start"); expect(start).toContain("nemoclaw_runtime_state_mutation_gate resume"); - expect(start).toContain("nemoclaw_runtime_state_mutation_acknowledge_release"); - expect(start).toMatch( - /nemoclaw_runtime_state_mutation_gate resume; then\n\s+if nemoclaw_runtime_state_mutation_acknowledge_release; then\n\s+return 0/u, - ); - expect(start).toContain('NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV="/usr/bin/mv"'); - expect(start).toContain( - '"$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV" -f -- "$pending" "$final"', - ); const startupGate = fs.readFileSync(STARTUP_GATE, "utf8"); expect(startupGate).toContain('DURABLE_DIRECTORY = "/var/lib/nemoclaw/runtime-state-mutation"'); expect(startupGate).toContain('"permitted": 10'); expect(startupGate).toContain('"activation-ready": 11'); expect(startupGate).toContain('"retry": 12'); - expect(startupGate).toContain( - 'RELEASE_ACK_PROTOCOL = "nemoclaw-runtime-state-mutation-release-ack-v1"', - ); - expect(startupGate).toContain('["acknowledge"]'); expect(fs.readFileSync(PUBLISHER, "utf8")).toContain( 'PYTHON_PATH = "/opt/hermes/.venv/bin/python3"', ); From 7f91eb7b88c2fb68ccc5f34224558f95974c6147 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 15:43:58 +0700 Subject: [PATCH 18/44] test(runtime): refresh Shields merge coverage Signed-off-by: San Dang --- .../sandbox-gpu-create-identity-gate.test.ts | 103 ------------------ ...runtime-state-mutation-release-ack.test.ts | 2 +- 2 files changed, 1 insertion(+), 104 deletions(-) diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 90b81539134..a3c67d9e915 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -354,109 +354,6 @@ describe("created sandbox identity gate", () => { expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); }); - it("waits for the exact created sandbox to appear through its owning gateway (#9833)", async () => { - let nonce = ""; - const input = noGpuInput(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { - nonce = createAttemptNonce(args); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - vi.mocked(deps.runOpenshell) - .mockReturnValueOnce({ - status: 1, - stdout: "", - stderr: - "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", - }) - .mockReturnValue({ - status: 0, - stdout: "Name: alpha\nId: alpha-sandbox-id\nState: Ready\n", - stderr: "", - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - - expect(deps.sleep).toHaveBeenCalledExactlyOnceWith(1); - expect(deps.runOpenshell).toHaveBeenNthCalledWith( - 1, - ["sandbox", "get", "-g", "nemoclaw", "alpha"], - expect.objectContaining({ ignoreError: true, suppressOutput: true }), - ); - expect(input.verifyCreatedSandboxBeforeEffects).toHaveBeenCalledOnce(); - }); - - it("rejects a different owner-scoped sandbox identity before post-create effects (#9833)", async () => { - let nonce = ""; - const input = noGpuInput(); - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { - nonce = createAttemptNonce(args); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - vi.mocked(deps.runOpenshell).mockReturnValue({ - status: 0, - stdout: "Name: alpha\nId: replacement-sandbox-id\nState: Ready\n", - stderr: "", - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "changed identity before policy verification", - ); - - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - }); - - it("stops when owner-scoped sandbox publication exceeds the deadline (#9833)", async () => { - let nonce = ""; - const input = noGpuInput(); - input.sandboxReadyTimeoutSecs = 0.001; - input.verifyCreatedSandboxBeforeEffects = vi.fn(); - input.revalidateVerifiedSandboxBeforeEffect = vi.fn(); - const patch = createGpuPatchFixture(); - mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); - mocks.streamSandboxCreate.mockImplementation(async (_command, args) => { - nonce = createAttemptNonce(args); - return { status: 0, output: "Created sandbox: alpha", sawProgress: true }; - }); - const deps = createGpuFlowDeps(); - vi.mocked(deps.runCaptureOpenshell).mockImplementationOnce(() => - sandboxListJson("alpha-sandbox-id", { [NEMOCLAW_CREATE_ATTEMPT_LABEL]: nonce }), - ); - vi.mocked(deps.runOpenshell).mockReturnValue({ - status: 1, - stdout: "", - stderr: - "Error: × code: 'Some requested entity was not found', message: \"sandbox not found\"", - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow( - "did not become visible through its owning gateway before policy verification", - ); - - expect(input.verifyCreatedSandboxBeforeEffects).not.toHaveBeenCalled(); - expect(patch.exitOnPatchError).not.toHaveBeenCalled(); - expect(patch.ensureApplied).not.toHaveBeenCalled(); - expect(mocks.waitForCreatedSandboxReadyWithTrace).not.toHaveBeenCalled(); - }); - it("rejects a same-name replacement before post-create effects (#9833)", async () => { let nonce = ""; const outputCanary = "replacement-output-must-not-be-reported"; diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index 8c9f705c6df..e6e00da9e39 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -37,7 +37,7 @@ start = control.ProcessReference( 12, 13, ) -fence = control.FenceProof(start, start, (os.geteuid(),)) +fence = control.FenceProof(start, start, (), (os.geteuid(),)) marker = {"transactionId": "c" * 64, "nonce": "b" * 64} release_payload = b'{"release":"exact"}\n' From c203f10f5bd52aaeaa2b4c9e6b0b2de93c31f07b Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 16:02:05 +0700 Subject: [PATCH 19/44] test(runtime): cover release acknowledgement order Signed-off-by: San Dang --- test/helpers/runtime-state-mutation-control-harness.ts | 1 + test/state/runtime-state-mutation-control.test.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index c1c873f2094..93c001221f8 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -973,6 +973,7 @@ with tempfile.TemporaryDirectory() as root: release_states = {1: "T", 10: "T", 75: "T", 76: "T", 77: "T", 78: "T"} release_events = [] +control._wait_for_startup_release_ack = lambda *_args: release_events.append(["release-ack"]) def state_process(original): return control.ProcessIdentity( original.pid, diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index c37025b848c..737ea3df605 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -204,7 +204,7 @@ describe("runtime state mutation controller", () => { }); }); - it("records release intent before resuming exact writers and leaves PID1 to host authority (#9485)", () => { + it("records release intent before resuming exact writers and waits for parent acknowledgement (#10155)", () => { expect(harnessResult).toMatchObject({ release: "activation-proven", released_marker: true, @@ -224,11 +224,13 @@ describe("runtime state mutation controller", () => { ["resume", 10], ["resume", 1], ["health"], + ["release-ack"], ]); expect(harnessResult.release_retry_events).toEqual([ ["verify-checkpoint"], ["release-receipt"], ["health"], + ["release-ack"], ]); expect(harnessResult.transient_exit_release_events).toEqual([ ["verify-checkpoint"], @@ -239,6 +241,7 @@ describe("runtime state mutation controller", () => { ["resume", 10], ["resume", 1], ["health"], + ["release-ack"], ]); expect(harnessResult.persistent_exit_release).toBe("activation-process-drift"); const events = harnessResult.state_events as unknown[][]; From 1bec8ca8799434cc3777e21d57731c6ca9a049ea Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 18:08:04 +0700 Subject: [PATCH 20/44] fix(runtime): rescan replaced Hermes writers Signed-off-by: San Dang --- scripts/runtime-state-mutation-control.py | 10 +++++++++- .../runtime-state-mutation-control-harness.ts | 17 +++++++++++++++++ .../runtime-state-mutation-control.test.ts | 5 +++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 94d14245d28..323d01b0ec4 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -2798,7 +2798,15 @@ def _exclude_writers( for writer in unexpected: identity = (writer.pid, writer.start_identity, requested) if identity not in signalled: - _signal_exact_process(writer, requested) + try: + _signal_exact_process(writer, requested) + except ControlError as error: + if error.code != "writer-pid-reused": + raise + # The pidfd check proved that this captured writer no longer + # owns the PID. Do not signal its replacement; the next scan + # authenticates the replacement as a separate writer. + continue signalled.add(identity) time.sleep(POLL_SECONDS) diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index 93c001221f8..5286c7067e4 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -618,6 +618,23 @@ control.KILL_SECONDS = 5 real_exclude_writers((1000, 1001), (control._process_reference(stopped_start),)) results["writer_signals"] = writer_signals results["writer_scans_remaining"] = len(scans) + +pid_reuse_scans = [ + (stopped_start, intruder), + (stopped_start,), + (stopped_start,), + (stopped_start,), +] +pid_reuse_signals = [] +control._capture_writer_processes = lambda _uids: pid_reuse_scans.pop(0) +def reject_replaced_writer(selected, requested): + pid_reuse_signals.append([selected.pid, requested]) + raise control.ControlError("writer-pid-reused") +control._signal_exact_process = reject_replaced_writer +real_exclude_writers((1000, 1001), (control._process_reference(stopped_start),)) +results["writer_pid_reuse_signals"] = pid_reuse_signals +results["writer_pid_reuse_scans_remaining"] = len(pid_reuse_scans) + control._capture_writer_processes = lambda _uids: (intruder,) control.TERM_SECONDS = 0 control.KILL_SECONDS = 0 diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 737ea3df605..fd421d55f73 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -94,6 +94,11 @@ describe("runtime state mutation controller", () => { expect(harnessResult.unknown_writer).toBe("unreadable-writer-process"); }); + it("rescans when an unexpected writer changes identity before signalling (#10155)", () => { + expect(harnessResult.writer_pid_reuse_signals).toEqual([[42, 15]]); + expect(harnessResult.writer_pid_reuse_scans_remaining).toBe(0); + }); + it("publishes, rolls an activated fence back, and recovers every durable phase (#7744)", () => { expect(harnessResult).toMatchObject({ publish: "published", From c63f95c778b24848201f440d7eb43ac02d5006a5 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 18:37:53 +0700 Subject: [PATCH 21/44] fix(runtime): wait for Hermes release publisher Signed-off-by: San Dang --- agents/hermes/start.sh | 48 ++----------------- scripts/runtime-state-mutation-control.py | 42 ++++++++++++++++ .../runtime-state-mutation-startup-gate.py | 28 +++++------ ...me-state-mutation-hermes-publisher.test.ts | 17 +++---- ...runtime-state-mutation-release-ack.test.ts | 30 +++++++++++- ...untime-state-mutation-startup-gate.test.ts | 10 +--- 6 files changed, 99 insertions(+), 76 deletions(-) diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index b14d109d188..ecf89f35b1f 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -30,17 +30,10 @@ NEMOCLAW_RUNTIME_STATE_MUTATION_RETRY_ARGV=("$@") readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON="/opt/hermes/.venv/bin/python3" readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER="/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py" readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV="/usr/bin/setpriv" -readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV="/usr/bin/mv" -readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP="/usr/bin/mktemp" -readonly NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM="/usr/bin/rm" -readonly NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT="/run/nemoclaw/runtime-state-mutation-startup" if [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" ] \ || [ ! -f "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \ || [ -L "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" ] \ - || [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV" ] \ - || [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP" ] \ - || [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" ] \ || { [ "$EUID" -eq 0 ] && [ ! -x "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" ]; }; then printf '%s\n' '[SECURITY] Required runtime state mutation startup gate is unavailable.' >&2 exit 1 @@ -60,48 +53,15 @@ nemoclaw_runtime_state_mutation_gate() { } nemoclaw_runtime_state_mutation_acknowledge_release() { - local nonce nonce_file pending final - nonce_file="$("$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP" \ - /tmp/nemoclaw-runtime-state-mutation-ack.XXXXXXXXXX)" || return 1 - if [ ! -f "$nonce_file" ] || [ -L "$nonce_file" ]; then - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" - return 1 - fi if [ "$EUID" -eq 0 ]; then "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" \ --reuid=sandbox --regid=sandbox --init-groups -- \ "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >"$nonce_file" || { - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" - return 1 - } - else - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >"$nonce_file" || { - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" - return 1 - } - fi - if ! IFS= read -r nonce <"$nonce_file"; then - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" - return 1 - fi - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM" -f -- "$nonce_file" - if [ "${#nonce}" -ne 64 ]; then - return 1 - fi - case "$nonce" in - *[!0-9a-f]*) return 1 ;; - esac - pending="${NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT}/${nonce}/.release-ack.json.pending" - final="${NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT}/${nonce}/release-ack.json" - if [ -f "$final" ] && [ ! -L "$final" ]; then - return 0 - fi - if [ ! -f "$pending" ] || [ -L "$pending" ]; then - return 1 + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >/dev/null + return fi - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV" -f -- "$pending" "$final" + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ + "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >/dev/null } nemoclaw_runtime_state_mutation_retry_exec() { diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 323d01b0ec4..a6c21ad819b 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -116,6 +116,8 @@ HERMES_CONFIG_GENERATION_PATH = "/sandbox/.hermes/.config-hash" START_LOG_PATH = b"/tmp/nemoclaw-start.log" START_LOG_DRAIN_PATHS = (b"tee", b"/usr/bin/tee", b"/bin/tee") +STARTUP_GATE_PYTHON = b"/opt/hermes/.venv/bin/python3" +STARTUP_GATE_HELPER = b"/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py" TRANSPORT_BROKER_BOOTSTRAP = ( b"import base64,sys,zlib;source=zlib.decompress(base64.b64decode(sys.argv.pop(1)),-15);" b"exec(compile(source,'','exec'))" @@ -2143,6 +2145,11 @@ def _wait_for_startup_release_ack( while True: if _read_startup_release_ack(marker, fence, release_payload) is not None: _recapture_reference(fence.start, "activation-release-identity-drift") + # The gate publishes the acknowledgement before its direct child + # returns to the startup shell. Keep this transaction active until + # that child exits so a following fence cannot stop the parent and + # terminate the publisher before the parent receives success. + _wait_for_release_ack_publisher(fence) return remaining = deadline - time.monotonic() if remaining <= 0: @@ -2150,6 +2157,41 @@ def _wait_for_startup_release_ack( time.sleep(min(POLL_SECONDS, remaining)) +def _is_release_ack_publisher( + process: ProcessIdentity, start: ProcessReference +) -> bool: + return bool( + process.parent_pid == start.pid + and process.uids == start.uids + and process.command + == ( + STARTUP_GATE_PYTHON, + b"-I", + STARTUP_GATE_HELPER, + b"acknowledge", + ) + ) + + +def _wait_for_release_ack_publisher(fence: FenceProof) -> None: + deadline = time.monotonic() + PROCESS_STATE_SECONDS + while True: + _recapture_reference(fence.start, "activation-release-identity-drift") + publishers = tuple( + process + for process in _capture_writer_processes(fence.writer_uids) + if _is_release_ack_publisher(process, fence.start) + ) + if not publishers: + return + if len(publishers) != 1: + _fail("activation-release-ack-invalid") + remaining = deadline - time.monotonic() + if remaining <= 0: + _fail("activation-release-ack-timeout") + time.sleep(min(POLL_SECONDS, remaining)) + + def _verify_activation_checkpoint( marker: dict[str, object], fence: FenceProof, activation: ActivationProof ) -> None: diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index dc3caceeb5c..3e3e7056fe7 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -36,7 +36,6 @@ CANDIDATE_NAME = "startup-complete.json" RETRY_ACK_NAME = "retry-ack.json" RELEASE_ACK_NAME = "release-ack.json" -RELEASE_ACK_PENDING_NAME = ".release-ack.json.pending" PERMIT_PROTOCOL = "nemoclaw-runtime-state-mutation-activation-permit-v1" RELEASE_PROTOCOL = "nemoclaw-runtime-state-mutation-activation-release-v1" RETRY_PROTOCOL = "nemoclaw-runtime-state-mutation-activation-retry-v1" @@ -641,19 +640,18 @@ def _prepare_release_ack(binding: dict[str, object]) -> str: "start": binding["start"], } payload = _canonical(ack) + b"\n" - for name in (RELEASE_ACK_NAME, RELEASE_ACK_PENDING_NAME): - existing = _read_at( - directory_fd, - name, - uid=os.geteuid(), - gid=os.getegid(), - mode=0o600, - missing=True, - ) - if existing is not None: - if existing != payload: - _fail("release-ack-conflict") - return str(binding["nonce"]) + existing = _read_at( + directory_fd, + RELEASE_ACK_NAME, + uid=os.geteuid(), + gid=os.getegid(), + mode=0o600, + missing=True, + ) + if existing is not None: + if existing != payload: + _fail("release-ack-conflict") + return str(binding["nonce"]) temporary = f".{RELEASE_ACK_NAME}.{os.getpid()}.{secrets.token_hex(8)}" fd = os.open( temporary, @@ -674,7 +672,7 @@ def _prepare_release_ack(binding: dict[str, object]) -> str: os.close(fd) os.replace( temporary, - RELEASE_ACK_PENDING_NAME, + RELEASE_ACK_NAME, src_dir_fd=directory_fd, dst_dir_fd=directory_fd, ) diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index 6e14f9bb4cc..c982a6f902e 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -333,7 +333,9 @@ function shellFunction(source: string, name: string, nextName: string): string { const start = source.indexOf(`${name}() {`); const end = source.indexOf(`\n${nextName}() {`, start); expect(start, `Expected ${name} in agents/hermes/start.sh`).toBeGreaterThanOrEqual(0); - expect(end, `Expected ${nextName} after ${name} in agents/hermes/start.sh`).toBeGreaterThan(start); + expect(end, `Expected ${nextName} after ${name} in agents/hermes/start.sh`).toBeGreaterThan( + start, + ); return source.slice(start, end); } @@ -411,7 +413,7 @@ describe("Hermes runtime state mutation publisher", () => { const waited = path.join(temporary, "controller-wait-finished"); fs.writeFileSync( gate, - `import json\nimport os\nimport sys\n\nexpected = int(os.environ["NEMOCLAW_TEST_EXPECTED_START_PID"])\nif os.getppid() != expected:\n raise SystemExit("gate-start-mismatch")\nnonce = os.environ["NEMOCLAW_TEST_NONCE"]\nroot = os.environ["NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT"]\ndirectory = os.path.join(root, nonce)\nos.mkdir(directory, 0o700)\npending = os.path.join(directory, ".release-ack.json.pending")\nwith open(pending, "x", encoding="utf-8") as stream:\n json.dump({"nonce": nonce}, stream, separators=(",", ":"))\nos.chmod(pending, 0o600)\nprint(nonce)\n`, + `import json\nimport os\nimport sys\n\nexpected = int(os.environ["NEMOCLAW_TEST_EXPECTED_START_PID"])\nif os.getppid() != expected:\n raise SystemExit("gate-start-mismatch")\nnonce = os.environ["NEMOCLAW_TEST_NONCE"]\nroot = os.environ["NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT"]\ndirectory = os.path.join(root, nonce)\nos.mkdir(directory, 0o700)\nfinal = os.path.join(directory, "release-ack.json")\nwith open(final, "x", encoding="utf-8") as stream:\n json.dump({"nonce": nonce}, stream, separators=(",", ":"))\nos.chmod(final, 0o600)\nprint(nonce)\n`, { mode: 0o700 }, ); const start = fs.readFileSync(START, "utf8"); @@ -439,9 +441,6 @@ wait "$controller_pid" NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON: "python3", NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER: gate, NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV: "setpriv", - NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MV: "mv", - NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_MKTEMP: "mktemp", - NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_RM: "rm", NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT: temporary, NEMOCLAW_TEST_CONTROLLER_WAIT: controllerWait, NEMOCLAW_TEST_NONCE: nonce, @@ -449,10 +448,12 @@ wait "$controller_pid" }, }); expect(result.status, result.stderr).toBe(0); - expect( - fs.readFileSync(path.join(temporary, nonce, "release-ack.json"), "utf8"), - ).toBe(`{"nonce":"${nonce}"}`); + expect(fs.readFileSync(path.join(temporary, nonce, "release-ack.json"), "utf8")).toBe( + `{"nonce":"${nonce}"}`, + ); expect(fs.existsSync(waited)).toBe(true); + expect(acknowledge).not.toContain("mktemp"); + expect(acknowledge).not.toContain("mv"); } finally { fs.rmSync(temporary, { force: true, recursive: true }); } diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index e6e00da9e39..1e902ca7fad 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -128,6 +128,32 @@ with tempfile.TemporaryDirectory() as root: release_payload, ) ) + publisher = control.ProcessIdentity( + 22, + "S", + start.pid, + "202", + start.uids, + ( + control.STARTUP_GATE_PYTHON, + b"-I", + control.STARTUP_GATE_HELPER, + b"acknowledge", + ), + 14, + 15, + ) + publisher_scans = {"count": 0} + def capture_publishers(_uids): + publisher_scans["count"] += 1 + return (publisher,) if publisher_scans["count"] == 1 else () + control._capture_writer_processes = capture_publishers + control._recapture_reference = lambda *_args: None + control.POLL_SECONDS = 0 + results["publisherWait"] = code( + lambda: control._wait_for_release_ack_publisher(fence) + ) + results["publisherScans"] = publisher_scans["count"] os.unlink(control.STARTUP_RELEASE_ACK_NAME, dir_fd=directory_fd) record_rejection( directory_fd, @@ -167,7 +193,7 @@ print(json.dumps(results, sort_keys=True)) `; describe("runtime state mutation release acknowledgement", () => { - it("accepts only a committed acknowledgement for the exact activation release (#10155)", () => { + it("accepts the exact acknowledgement after its startup child exits (#10155)", () => { const result = spawnSync("python3", ["-I", "-c", HARNESS, CONTROLLER], { encoding: "utf8", }); @@ -176,6 +202,8 @@ describe("runtime state mutation release acknowledgement", () => { cleaned: true, committed: "ok", pendingIgnored: true, + publisherScans: 2, + publisherWait: "ok", wrongNonce: "activation-release-ack-invalid", wrongRelease: "activation-release-ack-invalid", wrongStart: "activation-release-ack-invalid", diff --git a/test/state/runtime-state-mutation-startup-gate.test.ts b/test/state/runtime-state-mutation-startup-gate.test.ts index 263a2de6f3e..c9083f1281a 100644 --- a/test/state/runtime-state-mutation-startup-gate.test.ts +++ b/test/state/runtime-state-mutation-startup-gate.test.ts @@ -151,15 +151,9 @@ with tempfile.TemporaryDirectory() as root: results["release_ack_write_failure"] = code(lambda: gate._run("acknowledge")) gate.os.replace = original_replace results["release_ack_nonce"] = gate._run("acknowledge") - release_ack_pending = os.path.join( - candidate_directory, gate.RELEASE_ACK_PENDING_NAME - ) - with open(release_ack_pending, "rb") as stream: + release_ack_path = os.path.join(candidate_directory, gate.RELEASE_ACK_NAME) + with open(release_ack_path, "rb") as stream: results["release_ack"] = json.load(stream) - os.replace( - release_ack_pending, - os.path.join(candidate_directory, gate.RELEASE_ACK_NAME), - ) results["release_ack_committed"] = gate._run("acknowledge") gate._capture_parent = lambda: {**start, "pid": 42} results["foreign_parent_ack"] = code(lambda: gate._run("acknowledge")) From e74baa611e4ae8243b267ad61801e2819a7be2bc Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 18:54:06 +0700 Subject: [PATCH 22/44] fix(runtime): stabilize Hermes release observation Signed-off-by: San Dang --- scripts/runtime-state-mutation-control.py | 48 +++++++++++-------- .../runtime-state-mutation-control-harness.ts | 21 ++++++++ .../runtime-state-mutation-control.test.ts | 3 ++ ...runtime-state-mutation-release-ack.test.ts | 8 +++- 4 files changed, 58 insertions(+), 22 deletions(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index a6c21ad819b..1ba45f721a5 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -2175,6 +2175,7 @@ def _is_release_ack_publisher( def _wait_for_release_ack_publisher(fence: FenceProof) -> None: deadline = time.monotonic() + PROCESS_STATE_SECONDS + stable_absence = 0 while True: _recapture_reference(fence.start, "activation-release-identity-drift") publishers = tuple( @@ -2183,7 +2184,12 @@ def _wait_for_release_ack_publisher(fence: FenceProof) -> None: if _is_release_ack_publisher(process, fence.start) ) if not publishers: - return + stable_absence += 1 + if stable_absence >= STABLE_SCANS: + return + time.sleep(POLL_SECONDS) + continue + stable_absence = 0 if len(publishers) != 1: _fail("activation-release-ack-invalid") remaining = deadline - time.monotonic() @@ -2698,17 +2704,17 @@ def _discover_fence(expected_mount_namespace: str) -> FenceProof: if len(second_starts) == 1 else () ) - if ( - len(second_starts) != 1 - or not _process_matches_reference(second_starts[0], start_reference) - or len(second_support) != len(support_references) - or any( - not _process_matches_reference(process, reference) - for process, reference in zip(second_support, support_references) - ) - or not _is_openshell_supervisor(second_supervisor) - ): + if not _is_openshell_supervisor(second_supervisor): _fail("supervisor-identity-drift") + if len(second_starts) != 1 or not _process_matches_reference( + second_starts[0], start_reference + ): + _fail("start-process-identity-drift") + if len(second_support) != len(support_references) or any( + not _process_matches_reference(process, reference) + for process, reference in zip(second_support, support_references) + ): + _fail("startup-support-identity-drift") return FenceProof( supervisor_reference, start_reference, @@ -2879,19 +2885,21 @@ def _prove_fence_shape( ): _fail("supervisor-identity-drift") supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") - start = _recapture_reference(fence.start, "supervisor-identity-drift") + start = _recapture_reference(fence.start, "start-process-identity-drift") support = tuple( - _recapture_reference(reference, "supervisor-identity-drift") + _recapture_reference(reference, "startup-support-identity-drift") for reference in fence.start_support ) sandbox_uid = _sandbox_uid() - if not _is_openshell_supervisor(supervisor) or not _is_nemoclaw_start( - start, sandbox_uid - ) or any( + if not _is_openshell_supervisor(supervisor): + _fail("supervisor-identity-drift") + if not _is_nemoclaw_start(start, sandbox_uid): + _fail("start-process-identity-drift") + if any( not _is_start_log_drain(process, start, sandbox_uid) for process in support ): - _fail("supervisor-identity-drift") + _fail("startup-support-identity-drift") return supervisor, start @@ -3813,7 +3821,7 @@ def _activation_tree( by_pid = {process.pid: process for process in writers} start = by_pid.get(fence.start.pid) if start is None or not _process_matches_reference(start, fence.start): - _fail("supervisor-identity-drift") + _fail("start-process-identity-drift") tree: list[ProcessIdentity] = [] for process in writers: if process.pid == fence.start.pid: @@ -3901,7 +3909,7 @@ def _wait_for_startup_checkpoint(marker: dict[str, object], fence: FenceProof) - deadline = time.monotonic() + ACTIVATION_SECONDS while True: supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") - start = _recapture_reference(fence.start, "supervisor-identity-drift") + start = _recapture_reference(fence.start, "start-process-identity-drift") selected = _read_startup_candidate(marker, fence, required=False) if selected is not None and start.state in ("T", "t"): if supervisor.state not in ("T", "t"): @@ -4227,7 +4235,7 @@ def _prove_released_activation( process = _recapture_reference(reference, "activation-process-drift") if process.state in ("T", "t"): _fail("activation-process-stopped") - start = _recapture_reference(fence.start, "supervisor-identity-drift") + start = _recapture_reference(fence.start, "start-process-identity-drift") if start.state in ("T", "t"): _fail("start-process-stopped") supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index 5286c7067e4..2d13f965c13 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -566,6 +566,27 @@ try: finally: os.readlink = real_readlink +def fence_drift(missing_pid): + control._capture_process = lambda pid: { + 1: pid1, + 10: start, + 75: stdout_drain, + 76: stderr_drain, + }.get(pid) if pid != missing_pid else None + control.os.readlink = lambda selected: ( + "mnt:[401]" + if selected == control.MOUNT_NAMESPACE_PATH + else real_readlink(selected) + ) + try: + return code(lambda: real_prove_fence_shape(fence, "mnt:[401]")) + finally: + control.os.readlink = real_readlink + +results["supervisor_identity_drift"] = fence_drift(1) +results["start_identity_drift"] = fence_drift(10) +results["startup_support_identity_drift"] = fence_drift(75) + hold_events = [] control._prove_fence_shape = lambda _fence, _mount: (pid1, start) control._stop_reference = lambda reference: hold_events.append(["stop", reference.pid]) diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index fd421d55f73..2cc340c4ba9 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -65,6 +65,9 @@ describe("runtime state mutation controller", () => { discovered_pid1: 1, discovered_start: 10, wrong_pid1: "supervisor-unavailable", + supervisor_identity_drift: "supervisor-identity-drift", + start_identity_drift: "start-process-identity-drift", + startup_support_identity_drift: "startup-support-identity-drift", running_supervisor_hold: "supervisor-not-host-stopped", }); expect(harnessResult.hold_events).toEqual([ diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index 1e902ca7fad..71535f8a8c7 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -146,7 +146,11 @@ with tempfile.TemporaryDirectory() as root: publisher_scans = {"count": 0} def capture_publishers(_uids): publisher_scans["count"] += 1 - return (publisher,) if publisher_scans["count"] == 1 else () + return ( + (publisher,) + if publisher_scans["count"] in (1, 3) + else () + ) control._capture_writer_processes = capture_publishers control._recapture_reference = lambda *_args: None control.POLL_SECONDS = 0 @@ -202,7 +206,7 @@ describe("runtime state mutation release acknowledgement", () => { cleaned: true, committed: "ok", pendingIgnored: true, - publisherScans: 2, + publisherScans: 6, publisherWait: "ok", wrongNonce: "activation-release-ack-invalid", wrongRelease: "activation-release-ack-invalid", From 098abe9a2ff004d3a8c1121fd8c22a9590d52113 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 19:16:00 +0700 Subject: [PATCH 23/44] test(runtime): stabilize helper timeout allowance Signed-off-by: San Dang --- test/helpers/docker-state-mutation-harness.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index 589c8465f69..b3e76cc3656 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -362,7 +362,7 @@ function createContainerStateMutationHarness( "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", action, ], - Math.max(1, helperDeadline - Date.now()), + helperTimeout, request, ); if (helperResult.status !== null && helperResult.status < 0) { From 76ce503f99bd74f932a4b84ddfc253a77aca0f2b Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 19:51:43 +0700 Subject: [PATCH 24/44] fix(runtime): require Hermes parent release acknowledgement Signed-off-by: San Dang --- agents/hermes/start.sh | 4 + scripts/runtime-state-mutation-control.py | 92 +++++++++++++++---- .../runtime-state-mutation-startup-gate.py | 5 + .../runtime-state-mutation-control-harness.ts | 21 ++++- .../runtime-state-mutation-control.test.ts | 7 +- ...me-state-mutation-hermes-publisher.test.ts | 8 ++ ...runtime-state-mutation-release-ack.test.ts | 36 ++++++-- ...untime-state-mutation-startup-gate.test.ts | 23 +++++ 8 files changed, 164 insertions(+), 32 deletions(-) diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index ecf89f35b1f..5ee08d18a05 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -119,6 +119,10 @@ nemoclaw_runtime_state_mutation_checkpoint() { kill -STOP "$$" if nemoclaw_runtime_state_mutation_gate resume; then if nemoclaw_runtime_state_mutation_acknowledge_release; then + # The acknowledgement helper stops itself after publishing. The root + # controller resumes that exact child, then this parent stops only after + # Bash has reaped it and observed success. + kill -STOP "$$" return 0 fi printf '%s\n' '[SECURITY] Runtime state mutation release acknowledgement failed; holding startup.' >&2 diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 1ba45f721a5..de1302bf870 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -1375,7 +1375,7 @@ def _receipt_payload(marker: dict[str, object]) -> dict[str, object]: def _released_receipt_payload( marker: dict[str, object], completed_ledger_sha256: str, release_state: str ) -> dict[str, object]: - if release_state not in ("intent", "complete"): + if release_state not in ("intent", "acknowledged", "complete"): _fail("released-receipt-invalid") return { "schemaVersion": SCHEMA_VERSION, @@ -1393,7 +1393,7 @@ def _validate_released_receipt(value: object) -> dict[str, object]: if ( type(receipt["schemaVersion"]) is not int or receipt["schemaVersion"] != SCHEMA_VERSION - or receipt["releaseState"] not in ("intent", "complete") + or receipt["releaseState"] not in ("intent", "acknowledged", "complete") ): _fail("released-receipt-invalid") marker = _validate_marker(receipt["marker"]) @@ -2145,10 +2145,9 @@ def _wait_for_startup_release_ack( while True: if _read_startup_release_ack(marker, fence, release_payload) is not None: _recapture_reference(fence.start, "activation-release-identity-drift") - # The gate publishes the acknowledgement before its direct child - # returns to the startup shell. Keep this transaction active until - # that child exits so a following fence cannot stop the parent and - # terminate the publisher before the parent receives success. + # The exact publisher stops after writing the acknowledgement. Its + # parent stops only after the controller resumes that child, Bash + # reaps it, and the parent observes success. _wait_for_release_ack_publisher(fence) return remaining = deadline - time.monotonic() @@ -2175,23 +2174,29 @@ def _is_release_ack_publisher( def _wait_for_release_ack_publisher(fence: FenceProof) -> None: deadline = time.monotonic() + PROCESS_STATE_SECONDS - stable_absence = 0 + resumed_publisher: ProcessReference | None = None while True: - _recapture_reference(fence.start, "activation-release-identity-drift") + start = _recapture_reference( + fence.start, "activation-release-identity-drift" + ) publishers = tuple( process for process in _capture_writer_processes(fence.writer_uids) if _is_release_ack_publisher(process, fence.start) ) - if not publishers: - stable_absence += 1 - if stable_absence >= STABLE_SCANS: - return - time.sleep(POLL_SECONDS) - continue - stable_absence = 0 - if len(publishers) != 1: + if len(publishers) > 1: _fail("activation-release-ack-invalid") + if publishers: + publisher = publishers[0] + if resumed_publisher is not None and not _process_matches_reference( + publisher, resumed_publisher + ): + _fail("activation-release-ack-invalid") + if publisher.state in ("T", "t") and resumed_publisher is None: + _signal_exact_process(publisher, signal.SIGCONT) + resumed_publisher = _process_reference(publisher) + elif not publishers and start.state in ("T", "t"): + return remaining = deadline - time.monotonic() if remaining <= 0: _fail("activation-release-ack-timeout") @@ -4153,7 +4158,7 @@ def _acquire( if released is not None and released["transactionId"] == request.transaction_id: _fail( "transaction-release-pending" - if released["releaseState"] == "intent" + if released["releaseState"] != "complete" else "transaction-already-released" ) fence = _discover_fence(mount_namespace) @@ -4250,6 +4255,31 @@ def _prove_released_activation( _prove_live_activation(marker, fence, service, activation) +def _prove_parent_acknowledged_activation( + marker: dict[str, object], fence: FenceProof, activation: ActivationProof +) -> None: + persistent = set(activation.persistent_pids) + for reference in activation.processes: + if reference.pid not in persistent: + continue + process = _recapture_reference(reference, "activation-process-drift") + if process.state in ("T", "t"): + _fail("activation-process-stopped") + start = _recapture_reference(fence.start, "start-process-identity-drift") + if start.state not in ("T", "t"): + _fail("activation-release-parent-running") + supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") + if supervisor.state in ("T", "t"): + _fail("supervisor-process-stopped") + service_reference = next( + process + for process in activation.processes + if process.pid == activation.service_pid + ) + service = _recapture_reference(service_reference, "activation-service-drift") + _prove_live_activation(marker, fence, service, activation) + + def _release_activation_hold(durable_fd: int, marker: dict[str, object]) -> None: fence = _fence_from_value(marker["fence"]) activation = _activation_from_marker(marker) @@ -4261,6 +4291,11 @@ def _release_activation_hold(durable_fd: int, marker: dict[str, object]) -> None _activation_release_payload(marker, fence, activation) ) _prove_fence_shape(fence, str(marker["mountNamespace"])) + acknowledged = _read_startup_release_ack( + marker, fence, release_payload + ) is not None + start = _recapture_reference(fence.start, "start-process-identity-drift") + parent_already_acknowledged = acknowledged and start.state in ("T", "t") persistent = set(activation.persistent_pids) for reference in activation.processes: if _reference_is_terminated(reference): @@ -4268,13 +4303,23 @@ def _release_activation_hold(durable_fd: int, marker: dict[str, object]) -> None _fail("activation-process-drift") continue _resume_reference(reference) - _resume_reference(fence.start) + if not parent_already_acknowledged: + _resume_reference(fence.start) # Resume the exact pinned OpenShell supervisor last. Until the proven # workload is live, keeping PID 1 stopped prevents it from advertising a # transient Ready state or admitting unrelated sandbox commands. _resume_reference(fence.supervisor) - _prove_released_activation(marker, fence, activation) _wait_for_startup_release_ack(marker, fence, release_payload) + _prove_parent_acknowledged_activation(marker, fence, activation) + + +def _resume_acknowledged_parent(marker: dict[str, object]) -> None: + fence = _fence_from_value(marker["fence"]) + activation = _activation_from_marker(marker) + if activation is None: + _fail("activation-marker-invalid") + _resume_reference(fence.start) + _prove_released_activation(marker, fence, activation) def _complete_released_receipt( @@ -4288,6 +4333,15 @@ def _complete_released_receipt( if receipt["releaseState"] == "intent": _assert_runtime_binding(marker) _release_activation_hold(durable_fd, marker) + receipt = _write_released_receipt( + durable_fd, + marker, + str(receipt["completedLedgerSha256"]), + "acknowledged", + ) + if receipt["releaseState"] == "acknowledged": + _assert_runtime_binding(marker) + _resume_acknowledged_parent(marker) receipt = _write_released_receipt( durable_fd, marker, diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index 3e3e7056fe7..3df202ff6da 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -19,6 +19,7 @@ import os import re import secrets +import signal import stat import sys from typing import NoReturn @@ -739,6 +740,10 @@ def main(argv: list[str] | None = None) -> int: state = _run(arguments[0]) print(state) if arguments[0] == "acknowledge": + # Keep the exact publisher inspectable until the root controller + # resumes it. Its parent cannot report success before this process + # exits and is reaped. + os.kill(os.getpid(), signal.SIGSTOP) return 0 return { "inactive": 0, diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index 2d13f965c13..75b373ab783 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -46,6 +46,8 @@ real_wait_for_reference_running = control._wait_for_reference_running real_prove_fence_shape = control._prove_fence_shape real_resume_reference = control._resume_reference real_prove_released_activation = control._prove_released_activation +real_prove_parent_acknowledged_activation = control._prove_parent_acknowledged_activation +real_resume_acknowledged_parent = control._resume_acknowledged_parent control._assert_private_procfs = lambda: None control._open_activation_guard_pidfd = lambda _reference: os.open( os.devnull, os.O_RDONLY @@ -404,6 +406,10 @@ with tempfile.TemporaryDirectory() as root: receipt = control._load_released_receipt(durable_fd) events.append(["release-hold", receipt["releaseState"]]) control._release_activation_hold = release_hold + def resume_acknowledged_parent(_marker): + receipt = control._load_released_receipt(durable_fd) + events.append(["release-parent-resume", receipt["releaseState"]]) + control._resume_acknowledged_parent = resume_acknowledged_parent try: active_value = acquire_value() active = parse("acquire", active_value) @@ -1011,7 +1017,15 @@ with tempfile.TemporaryDirectory() as root: release_states = {1: "T", 10: "T", 75: "T", 76: "T", 77: "T", 78: "T"} release_events = [] -control._wait_for_startup_release_ack = lambda *_args: release_events.append(["release-ack"]) +release_ack_exists = {"value": False} +control._read_startup_release_ack = lambda *_args: ( + {"acknowledged": True} if release_ack_exists["value"] else None +) +def wait_for_startup_release_ack(*_args): + release_events.append(["release-ack"]) + release_ack_exists["value"] = True + release_states[10] = "T" +control._wait_for_startup_release_ack = wait_for_startup_release_ack def state_process(original): return control.ProcessIdentity( original.pid, @@ -1045,6 +1059,9 @@ def resume_reference(reference): return state_process(by_release_pid[reference.pid]) control._resume_reference = resume_reference control._prove_released_activation = lambda *_args: release_events.append(["health"]) +control._prove_parent_acknowledged_activation = lambda *_args: release_events.append( + ["parent-ack-health"] +) control._verify_activation_checkpoint = lambda *_args: release_events.append(["verify-checkpoint"]) control._publish_activation_release = lambda *_args: release_events.append(["release-receipt"]) terminated_release_pids = set() @@ -1067,12 +1084,14 @@ with tempfile.TemporaryDirectory() as root: release_events.clear() release_states.update({1: "T", 10: "T", 75: "T", 76: "T", 77: "T", 78: "T"}) + release_ack_exists["value"] = False terminated_release_pids.add(78) real_release_activation_hold(durable_fd, marker_for_helpers) results["transient_exit_release_events"] = list(release_events) release_events.clear() release_states.update({1: "T", 10: "T", 75: "T", 76: "T", 77: "T", 78: "T"}) + release_ack_exists["value"] = False terminated_release_pids.clear() terminated_release_pids.add(77) results["persistent_exit_release"] = code( diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 2cc340c4ba9..db1425ecc3c 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -231,14 +231,14 @@ describe("runtime state mutation controller", () => { ["resume", 78], ["resume", 10], ["resume", 1], - ["health"], ["release-ack"], + ["parent-ack-health"], ]); expect(harnessResult.release_retry_events).toEqual([ ["verify-checkpoint"], ["release-receipt"], - ["health"], ["release-ack"], + ["parent-ack-health"], ]); expect(harnessResult.transient_exit_release_events).toEqual([ ["verify-checkpoint"], @@ -248,12 +248,13 @@ describe("runtime state mutation controller", () => { ["resume", 77], ["resume", 10], ["resume", 1], - ["health"], ["release-ack"], + ["parent-ack-health"], ]); expect(harnessResult.persistent_exit_release).toBe("activation-process-drift"); const events = harnessResult.state_events as unknown[][]; expect(events).toContainEqual(["release-hold", "intent"]); + expect(events).toContainEqual(["release-parent-resume", "acknowledged"]); expect(events).toContainEqual(["protocol-cleanup", "activation-proven"]); }); diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index c982a6f902e..0576ea12c2c 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -454,6 +454,14 @@ wait "$controller_pid" expect(fs.existsSync(waited)).toBe(true); expect(acknowledge).not.toContain("mktemp"); expect(acknowledge).not.toContain("mv"); + const checkpoint = start.indexOf( + "if nemoclaw_runtime_state_mutation_acknowledge_release; then", + ); + const parentStop = start.indexOf('kill -STOP "$$"', checkpoint); + const acknowledgedReturn = start.indexOf("return 0", parentStop); + expect(checkpoint).toBeGreaterThan(0); + expect(parentStop).toBeGreaterThan(checkpoint); + expect(acknowledgedReturn).toBeGreaterThan(parentStop); } finally { fs.rmSync(temporary, { force: true, recursive: true }); } diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index 71535f8a8c7..0d540e32cdd 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -16,6 +16,7 @@ import hashlib import importlib.util import json import os +import signal import sys import tempfile @@ -130,7 +131,7 @@ with tempfile.TemporaryDirectory() as root: ) publisher = control.ProcessIdentity( 22, - "S", + "T", start.pid, "202", start.uids, @@ -144,20 +145,35 @@ with tempfile.TemporaryDirectory() as root: 15, ) publisher_scans = {"count": 0} + publisher_signals = [] + start_state = {"value": "S"} def capture_publishers(_uids): publisher_scans["count"] += 1 - return ( - (publisher,) - if publisher_scans["count"] in (1, 3) - else () - ) + return (publisher,) if publisher_scans["count"] == 1 else () control._capture_writer_processes = capture_publishers - control._recapture_reference = lambda *_args: None + control._recapture_reference = lambda *_args: control.ProcessIdentity( + start.pid, + start_state["value"], + start.parent_pid, + start.start_identity, + start.uids, + (control.NEMOCLAW_START_PATH,), + start.proc_device, + start.proc_inode, + ) + def signal_publisher(selected, requested): + publisher_signals.append( + [selected.pid, requested == signal.SIGCONT] + ) + start_state["value"] = "T" + control._signal_exact_process = signal_publisher control.POLL_SECONDS = 0 results["publisherWait"] = code( lambda: control._wait_for_release_ack_publisher(fence) ) results["publisherScans"] = publisher_scans["count"] + results["publisherSignals"] = publisher_signals + results["parentStopped"] = start_state["value"] == "T" os.unlink(control.STARTUP_RELEASE_ACK_NAME, dir_fd=directory_fd) record_rejection( directory_fd, @@ -197,7 +213,7 @@ print(json.dumps(results, sort_keys=True)) `; describe("runtime state mutation release acknowledgement", () => { - it("accepts the exact acknowledgement after its startup child exits (#10155)", () => { + it("resumes the exact acknowledgement child and waits for its parent (#10155)", () => { const result = spawnSync("python3", ["-I", "-c", HARNESS, CONTROLLER], { encoding: "utf8", }); @@ -205,8 +221,10 @@ describe("runtime state mutation release acknowledgement", () => { expect(JSON.parse(result.stdout)).toEqual({ cleaned: true, committed: "ok", + parentStopped: true, pendingIgnored: true, - publisherScans: 6, + publisherScans: 2, + publisherSignals: [[22, true]], publisherWait: "ok", wrongNonce: "activation-release-ack-invalid", wrongRelease: "activation-release-ack-invalid", diff --git a/test/state/runtime-state-mutation-startup-gate.test.ts b/test/state/runtime-state-mutation-startup-gate.test.ts index c9083f1281a..2e2fdd7ab4e 100644 --- a/test/state/runtime-state-mutation-startup-gate.test.ts +++ b/test/state/runtime-state-mutation-startup-gate.test.ts @@ -18,6 +18,7 @@ import hashlib import importlib.util import json import os +import signal import sys import tempfile @@ -155,6 +156,26 @@ with tempfile.TemporaryDirectory() as root: with open(release_ack_path, "rb") as stream: results["release_ack"] = json.load(stream) results["release_ack_committed"] = gate._run("acknowledge") + publisher_pid = os.fork() + if publisher_pid == 0: + null_fd = os.open(os.devnull, os.O_WRONLY) + os.dup2(null_fd, 1) + os.dup2(null_fd, 2) + os.close(null_fd) + os._exit(gate.main(["acknowledge"])) + stopped_pid, stopped_status = os.waitpid(publisher_pid, os.WUNTRACED) + results["release_publisher_stopped"] = ( + stopped_pid == publisher_pid + and os.WIFSTOPPED(stopped_status) + and os.WSTOPSIG(stopped_status) == signal.SIGSTOP + ) + os.kill(publisher_pid, signal.SIGCONT) + resumed_pid, resumed_status = os.waitpid(publisher_pid, 0) + results["release_publisher_resumed"] = ( + resumed_pid == publisher_pid + and os.WIFEXITED(resumed_status) + and os.WEXITSTATUS(resumed_status) == 0 + ) gate._capture_parent = lambda: {**start, "pid": 42} results["foreign_parent_ack"] = code(lambda: gate._run("acknowledge")) gate._capture_parent = lambda: start @@ -217,6 +238,8 @@ describe("runtime state mutation startup gate", () => { release_ack_write_failure: "release-ack-write-failed", release_ack_nonce: "b".repeat(64), release_ack_committed: "b".repeat(64), + release_publisher_stopped: true, + release_publisher_resumed: true, foreign_parent_ack: "gate-start-mismatch", tampered_release: "release-candidate-mismatch", symlink_directory: "unsafe-directory", From 338257ccc6558e8d21971f6d813fb3362cb82560 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 20:14:24 +0700 Subject: [PATCH 25/44] fix(runtime): rescan vanished mutation processes Signed-off-by: San Dang --- scripts/runtime-state-mutation-control.py | 4 +++- .../runtime-state-mutation-control-harness.ts | 18 ++++++++++++++++++ .../runtime-state-mutation-control.test.ts | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index de1302bf870..c694cdcc1b7 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -2509,10 +2509,12 @@ def _capture_process(pid: int) -> ProcessIdentity | None: command_raw = _read_proc_file(os.path.join(process_path, "cmdline")) second_stat = _read_proc_file(os.path.join(process_path, "stat")) after = os.stat(process_path, follow_symlinks=False) - except FileNotFoundError: + except (FileNotFoundError, ProcessLookupError): return None except PermissionError: _fail("unreadable-writer-process") + except OSError: + _fail("unreadable-writer-process") _first_state, parent, start = _parse_proc_stat(pid, first_stat) second_state, second_parent, second_start = _parse_proc_stat(pid, second_stat) if ( diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index 75b373ab783..b8fa369a7fa 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -41,6 +41,7 @@ real_health_status = control._health_status real_release_activation_hold = control._release_activation_hold real_parse_proc_uids = control._parse_proc_uids real_recapture_reference = control._recapture_reference +real_capture_process = control._capture_process real_signal_exact_process = control._signal_exact_process real_wait_for_reference_running = control._wait_for_reference_running real_prove_fence_shape = control._prove_fence_shape @@ -245,6 +246,23 @@ activation = control.ActivationProof( ) results = {} +with tempfile.TemporaryDirectory() as process_probe: + process_metadata = os.stat(process_probe, follow_symlinks=False) + real_os_stat = control.os.stat + real_read_proc_file = control._read_proc_file + control.os.stat = lambda _path, follow_symlinks=False: process_metadata + try: + control._read_proc_file = lambda _path, _maximum=control.MAX_PROC_FILE_BYTES: ( + _ for _ in () + ).throw(ProcessLookupError()) + results["vanished_process"] = real_capture_process(42) is None + control._read_proc_file = lambda _path, _maximum=control.MAX_PROC_FILE_BYTES: ( + _ for _ in () + ).throw(OSError()) + results["unreadable_process_io"] = code(lambda: real_capture_process(42)) + finally: + control.os.stat = real_os_stat + control._read_proc_file = real_read_proc_file with tempfile.TemporaryDirectory() as atomic_root: atomic_root_fd = os.open(atomic_root, os.O_RDONLY | os.O_DIRECTORY) atomic_creation_modes = [] diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index db1425ecc3c..f53fd9c41e3 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -95,6 +95,8 @@ describe("runtime state mutation controller", () => { expect(harnessResult.writer_scans_remaining).toBe(0); expect(harnessResult.unstoppable_writer).toBe("writer-exclusion-timeout"); expect(harnessResult.unknown_writer).toBe("unreadable-writer-process"); + expect(harnessResult.vanished_process).toBe(true); + expect(harnessResult.unreadable_process_io).toBe("unreadable-writer-process"); }); it("rescans when an unexpected writer changes identity before signalling (#10155)", () => { From d6477346aa1abdc49f9762c662153a6f51b6680b Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 27 Aug 2026 20:32:10 +0700 Subject: [PATCH 26/44] fix(runtime): recapture exact mutation processes Signed-off-by: San Dang --- scripts/runtime-state-mutation-control.py | 22 ++++++++++++----- .../runtime-state-mutation-control-harness.ts | 24 +++++++++++++++++++ .../runtime-state-mutation-control.test.ts | 9 +++++++ ...runtime-state-mutation-release-ack.test.ts | 24 +++++++++++-------- 4 files changed, 63 insertions(+), 16 deletions(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index c694cdcc1b7..44758c6a5bd 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -2193,8 +2193,9 @@ def _wait_for_release_ack_publisher(fence: FenceProof) -> None: ): _fail("activation-release-ack-invalid") if publisher.state in ("T", "t") and resumed_publisher is None: - _signal_exact_process(publisher, signal.SIGCONT) - resumed_publisher = _process_reference(publisher) + publisher_reference = _process_reference(publisher) + _signal_reference(publisher_reference, signal.SIGCONT) + resumed_publisher = publisher_reference elif not publishers and start.state in ("T", "t"): return remaining = deadline - time.monotonic() @@ -2757,8 +2758,17 @@ def _signal_exact_process(process: ProcessIdentity, requested_signal: int) -> No def _signal_reference(reference: ProcessReference, requested_signal: int) -> None: - process = _recapture_reference(reference) - _signal_exact_process(process, requested_signal) + for attempt in range(2): + process = _recapture_reference(reference) + try: + _signal_exact_process(process, requested_signal) + return + except ControlError as error: + if error.code != "writer-pid-reused" or attempt > 0: + raise + # Re-open the persisted exact reference once. A real replacement + # fails recapture; a process that still matches can be signalled + # through a fresh pidfd without weakening the identity check. def _wait_for_reference_state( @@ -2778,7 +2788,7 @@ def _wait_for_reference_state( def _stop_reference(reference: ProcessReference) -> None: process = _recapture_reference(reference) if process.state not in ("T", "t"): - _signal_exact_process(process, signal.SIGSTOP) + _signal_reference(reference, signal.SIGSTOP) _wait_for_reference_state(reference, ("T", "t")) @@ -4228,7 +4238,7 @@ def _wait_for_reference_running(reference: ProcessReference) -> ProcessIdentity: def _resume_reference(reference: ProcessReference) -> ProcessIdentity: process = _recapture_reference(reference) if process.state in ("T", "t"): - _signal_exact_process(process, signal.SIGCONT) + _signal_reference(reference, signal.SIGCONT) return _wait_for_reference_running(reference) diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index b8fa369a7fa..2aa80c5283d 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -43,6 +43,7 @@ real_parse_proc_uids = control._parse_proc_uids real_recapture_reference = control._recapture_reference real_capture_process = control._capture_process real_signal_exact_process = control._signal_exact_process +real_signal_reference = control._signal_reference real_wait_for_reference_running = control._wait_for_reference_running real_prove_fence_shape = control._prove_fence_shape real_resume_reference = control._resume_reference @@ -263,6 +264,29 @@ with tempfile.TemporaryDirectory() as process_probe: finally: control.os.stat = real_os_stat control._read_proc_file = real_read_proc_file +reference_signal_attempts = [] +control._recapture_reference = lambda _reference, _code="fenced-process-drift": start +def signal_reference_after_rescan(selected, requested): + reference_signal_attempts.append([selected.pid, requested]) + if len(reference_signal_attempts) == 1: + raise control.ControlError("writer-pid-reused") +control._signal_exact_process = signal_reference_after_rescan +real_signal_reference(control._process_reference(start), signal.SIGSTOP) +results["reference_signal_attempts"] = reference_signal_attempts +replacement_recaptures = [start] +control._recapture_reference = lambda _reference, _code="fenced-process-drift": ( + replacement_recaptures.pop(0) + if replacement_recaptures + else (_ for _ in ()).throw(control.ControlError(_code)) +) +control._signal_exact_process = lambda _selected, _requested: (_ for _ in ()).throw( + control.ControlError("writer-pid-reused") +) +results["replaced_reference_signal"] = code( + lambda: real_signal_reference(control._process_reference(start), signal.SIGSTOP) +) +control._recapture_reference = real_recapture_reference +control._signal_exact_process = real_signal_exact_process with tempfile.TemporaryDirectory() as atomic_root: atomic_root_fd = os.open(atomic_root, os.O_RDONLY | os.O_DIRECTORY) atomic_creation_modes = [] diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index f53fd9c41e3..fdf086efd82 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -104,6 +104,15 @@ describe("runtime state mutation controller", () => { expect(harnessResult.writer_pid_reuse_scans_remaining).toBe(0); }); + it("recaptures an exact process reference before retrying a raced signal (#10155)", () => { + const sigstop = harnessResult.sigstop as number; + expect(harnessResult.reference_signal_attempts).toEqual([ + [10, sigstop], + [10, sigstop], + ]); + expect(harnessResult.replaced_reference_signal).toBe("fenced-process-drift"); + }); + it("publishes, rolls an activated fence back, and recovers every durable phase (#7744)", () => { expect(harnessResult).toMatchObject({ publish: "published", diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index 0d540e32cdd..e3e5070569f 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -151,16 +151,20 @@ with tempfile.TemporaryDirectory() as root: publisher_scans["count"] += 1 return (publisher,) if publisher_scans["count"] == 1 else () control._capture_writer_processes = capture_publishers - control._recapture_reference = lambda *_args: control.ProcessIdentity( - start.pid, - start_state["value"], - start.parent_pid, - start.start_identity, - start.uids, - (control.NEMOCLAW_START_PATH,), - start.proc_device, - start.proc_inode, - ) + def recapture(reference, *_args): + if reference.pid == publisher.pid: + return publisher + return control.ProcessIdentity( + start.pid, + start_state["value"], + start.parent_pid, + start.start_identity, + start.uids, + (control.NEMOCLAW_START_PATH,), + start.proc_device, + start.proc_inode, + ) + control._recapture_reference = recapture def signal_publisher(selected, requested): publisher_signals.append( [selected.pid, requested == signal.SIGCONT] From cceb629dc8e81a2fc3eae444d1c6c9ea5cd98aca Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 27 Aug 2026 11:27:18 -0700 Subject: [PATCH 27/44] fix(runtime): retry raced mutation signals --- scripts/runtime-state-mutation-control.py | 17 ++++++++++++----- .../runtime-state-mutation-control-harness.ts | 8 +++++++- .../runtime-state-mutation-control.test.ts | 5 ++++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 44758c6a5bd..c58a64a363b 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -2758,17 +2758,24 @@ def _signal_exact_process(process: ProcessIdentity, requested_signal: int) -> No def _signal_reference(reference: ProcessReference, requested_signal: int) -> None: - for attempt in range(2): + deadline = time.monotonic() + PROCESS_STATE_SECONDS + while True: process = _recapture_reference(reference) try: _signal_exact_process(process, requested_signal) return except ControlError as error: - if error.code != "writer-pid-reused" or attempt > 0: + if error.code != "writer-pid-reused": + raise + # Re-open the persisted exact reference within the existing + # process-state deadline. A real replacement fails recapture; a + # process that still matches can be signalled through a fresh + # pidfd without weakening the identity check. A busy restart can + # cross more than one short-lived PID snapshot before settling. + remaining = deadline - time.monotonic() + if remaining <= 0: raise - # Re-open the persisted exact reference once. A real replacement - # fails recapture; a process that still matches can be signalled - # through a fresh pidfd without weakening the identity check. + time.sleep(min(POLL_SECONDS, remaining)) def _wait_for_reference_state( diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index 2aa80c5283d..ea711f06eb7 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -268,7 +268,7 @@ reference_signal_attempts = [] control._recapture_reference = lambda _reference, _code="fenced-process-drift": start def signal_reference_after_rescan(selected, requested): reference_signal_attempts.append([selected.pid, requested]) - if len(reference_signal_attempts) == 1: + if len(reference_signal_attempts) < 4: raise control.ControlError("writer-pid-reused") control._signal_exact_process = signal_reference_after_rescan real_signal_reference(control._process_reference(start), signal.SIGSTOP) @@ -285,6 +285,12 @@ control._signal_exact_process = lambda _selected, _requested: (_ for _ in ()).th results["replaced_reference_signal"] = code( lambda: real_signal_reference(control._process_reference(start), signal.SIGSTOP) ) +control._recapture_reference = lambda _reference, _code="fenced-process-drift": start +control.PROCESS_STATE_SECONDS = 0 +results["reference_signal_timeout"] = code( + lambda: real_signal_reference(control._process_reference(start), signal.SIGSTOP) +) +control.PROCESS_STATE_SECONDS = 5 control._recapture_reference = real_recapture_reference control._signal_exact_process = real_signal_exact_process with tempfile.TemporaryDirectory() as atomic_root: diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index fdf086efd82..34698859e5d 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -104,13 +104,16 @@ describe("runtime state mutation controller", () => { expect(harnessResult.writer_pid_reuse_scans_remaining).toBe(0); }); - it("recaptures an exact process reference before retrying a raced signal (#10155)", () => { + it("keeps recapturing an exact process reference across raced signals (#10155)", () => { const sigstop = harnessResult.sigstop as number; expect(harnessResult.reference_signal_attempts).toEqual([ [10, sigstop], [10, sigstop], + [10, sigstop], + [10, sigstop], ]); expect(harnessResult.replaced_reference_signal).toBe("fenced-process-drift"); + expect(harnessResult.reference_signal_timeout).toBe("writer-pid-reused"); }); it("publishes, rolls an activated fence back, and recovers every durable phase (#7744)", () => { From 05aed98d91cb2f01d31a24c4bcdd89387eece28b Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 27 Aug 2026 12:13:24 -0700 Subject: [PATCH 28/44] fix(shields): observe settled Hermes lock --- src/lib/shields/index.ts | 77 +++++++++++++++--------- src/lib/shields/relock-reconfirm.test.ts | 13 ++++ src/lib/shields/relock-reconfirm.ts | 14 +++-- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index ce1bb499d1d..80745f51916 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -4030,6 +4030,38 @@ function captureSealHashes(sandboxName: string, filesToHash: string[]): { [path: return hashes; } +function verifyHermesProviderLockedPosture( + sandboxName: string, + target: AgentConfigTarget, +): { chattrApplied: boolean; fileHashes: { [path: string]: string } } { + const chattrApplied = hermesProviderChattrApplied(sandboxName, target); + const verified = verifyShieldsLockState(sandboxName, target, { + verifyChattr: chattrApplied, + verifyParentProtection: true, + exec: (cmd: string[]) => privilegedSandboxExecCapture(sandboxName, cmd), + assertLegacyLayout: assertNoLegacyStateLayout, + }); + if (verified.issues.length > 0) { + throw new Error(`Config not locked: ${verified.issues.join(", ")}`); + } + return { + chattrApplied, + fileHashes: captureSealHashes(sandboxName, [ + target.configPath, + ...(target.sensitiveFiles || []), + ]), + }; +} + +function hermesProviderLockConfirmation( + sandboxName: string, + target: AgentConfigTarget, + protocol: HermesShieldsProtocol, +): (() => { chattrApplied: boolean; fileHashes: { [path: string]: string } }) | undefined { + if (protocol !== "provider-state-mutation-v2") return undefined; + return () => verifyHermesProviderLockedPosture(sandboxName, target); +} + function lockAgentConfigUnderMutationLock( sandboxName: string, rawTarget: AgentConfigTarget, @@ -4038,36 +4070,17 @@ function lockAgentConfigUnderMutationLock( ): { chattrApplied: boolean; fileHashes: { [path: string]: string } } { const target = ensureConfigHashSensitiveFile(rawTarget); if (target.agentName === "hermes" && protocol === "provider-state-mutation-v2") { - const verifyProviderLockedPosture = () => { - const chattrApplied = hermesProviderChattrApplied(sandboxName, target); - const verified = verifyShieldsLockState(sandboxName, target, { - verifyChattr: chattrApplied, - verifyParentProtection: true, - exec: (cmd: string[]) => privilegedSandboxExecCapture(sandboxName, cmd), - assertLegacyLayout: assertNoLegacyStateLayout, - }); - if (verified.issues.length > 0) { - throw new Error(`Config not locked: ${verified.issues.join(", ")}`); - } - return { - chattrApplied, - fileHashes: captureSealHashes(sandboxName, [ - target.configPath, - ...(target.sensitiveFiles || []), - ]), - }; - }; if (rollbackLocked) { runHermesProviderProtectionTransition(sandboxName, target, "locked", "locked"); try { - return verifyProviderLockedPosture(); + return verifyHermesProviderLockedPosture(sandboxName, target); } catch { // Repair a drifted live posture with a mutable rollback. Locked-target // failures retain the provider fence, so this remains fail-closed. } } runHermesProviderProtectionTransition(sandboxName, target, "locked", "mutable"); - return verifyProviderLockedPosture(); + return verifyHermesProviderLockedPosture(sandboxName, target); } const compatibilityIssues = stateLockPlanCompatibilityIssues( stateDirLockExec(sandboxName), @@ -4904,8 +4917,9 @@ function rollbackShieldsDown( // the rolled-back config DRIFTED — same fail-closed treatment as the // auto-restore path. Leaves the hashes null (→ "manual intervention" // below) when the lock will not re-confirm. - const relock = relockAndReconfirm(() => - lockAgentConfigUnderMutationLock(sandboxName, target, true, protocol), + const relock = relockAndReconfirm( + () => lockAgentConfigUnderMutationLock(sandboxName, target, true, protocol), + { confirm: hermesProviderLockConfirmation(sandboxName, target, protocol) }, ); if (relock.ok && relock.lastResult) { rollbackChattrApplied = relock.lastResult.chattrApplied; @@ -5021,8 +5035,9 @@ function activateLockdownFromSnapshot( // which mark shields UP on this result — so a reconciler revert here would // otherwise leave the same DRIFTED state #4663 is about. relockAndReconfirm // fails closed (ok:false) when the lock will not hold past the settle window. - const relock = relockAndReconfirm(() => - lockAgentConfig(sandboxName, target, false, allowLegacyHermesProtocol, protocol), + const relock = relockAndReconfirm( + () => lockAgentConfig(sandboxName, target, false, allowLegacyHermesProtocol, protocol), + { confirm: hermesProviderLockConfirmation(sandboxName, target, protocol) }, ); if (!relock.ok || !relock.lastResult) { return { @@ -6407,8 +6422,16 @@ function shieldsUpWithoutHostLock(sandboxName: string, opts: ShieldsUpOpts = {}) // reverted on DGX Station / DGX Spark, leaving the sandbox DRIFTED. This // narrows (does not close) the revert window; the chattr +i immutable bit // applied inside lockAgentConfig is the only fully durable defense. - const relock = relockAndReconfirm(() => - lockAgentConfig(sandboxName, target, true, opts.allowLegacyHermesProtocol === true, protocol), + const relock = relockAndReconfirm( + () => + lockAgentConfig( + sandboxName, + target, + true, + opts.allowLegacyHermesProtocol === true, + protocol, + ), + { confirm: hermesProviderLockConfirmation(sandboxName, target, protocol) }, ); if (!relock.ok || !relock.lastResult) { const message = relock.error ?? "Config re-lock did not re-confirm after settle window"; diff --git a/src/lib/shields/relock-reconfirm.test.ts b/src/lib/shields/relock-reconfirm.test.ts index beabc6a03a9..030cf73de22 100644 --- a/src/lib/shields/relock-reconfirm.test.ts +++ b/src/lib/shields/relock-reconfirm.test.ts @@ -32,6 +32,19 @@ describe("relockAndReconfirm", () => { expect(sleep).toHaveBeenCalledWith(5); }); + it("observes the settled posture without applying the lock twice (#10155)", () => { + const lock = vi.fn(() => okResult()); + const confirm = vi.fn(() => okResult()); + const sleep = vi.fn(); + + const result = relockAndReconfirm(lock, { confirm, sleep, settleMs: 5 }); + + expect(result).toMatchObject({ ok: true, attempts: 1, lastResult: okResult() }); + expect(lock).toHaveBeenCalledOnce(); + expect(confirm).toHaveBeenCalledOnce(); + expect(sleep).toHaveBeenCalledWith(5); + }); + it("fails closed (bounded) when the re-confirm always throws after a clean apply", () => { // Apply succeeds every time, but the reconciler reverts before each // re-confirm so every re-confirm throws. diff --git a/src/lib/shields/relock-reconfirm.ts b/src/lib/shields/relock-reconfirm.ts index d616cedee32..3a7a018b727 100644 --- a/src/lib/shields/relock-reconfirm.ts +++ b/src/lib/shields/relock-reconfirm.ts @@ -13,7 +13,9 @@ // // `relockAndReconfirm` runs a bounded "lock -> settle -> re-confirm -> re-lock // if drifted" cycle and only declares the lock UP when a re-confirmation passes -// after the reconciler has had a chance to settle. +// after the reconciler has had a chance to settle. Callers whose lock operation +// restarts a runtime can supply a read-only confirmation instead of repeating +// that transition while the runtime is still settling. // // IMPORTANT — this NARROWS the race window, it does NOT close it. After the // final re-confirm returns, the same reconciler can revert perms one settle @@ -50,6 +52,8 @@ export interface RelockReconfirmOptions { settleMs?: number; /** Injectable synchronous sleep, for unit tests. Defaults to `sleepMs`. */ sleep?: (ms: number) => void; + /** Observe the settled lock without applying it again. Defaults to `lock`. */ + confirm?: LockFn; } export interface RelockReconfirmResult { @@ -96,8 +100,9 @@ export function resolveSettleMs(): number { * 1. `lock()` — apply + verify. If this throws, fail immediately (the lock * could not even be applied/verified once). * 2. `sleep(settleMs)` — give the gateway/reconciler time to settle. - * 3. `lock()` — re-confirm. Success => re-confirmed, return ok. Throw => the - * reconciler reverted during the settle window; retry the whole cycle. + * 3. `confirm()` — observe the settled posture (defaults to `lock`). Success + * => re-confirmed, return ok. Throw => the reconciler reverted during the + * settle window; retry the whole cycle. * * Returns ok:false (fail closed) when attempts are exhausted or the first * apply of an attempt throws. NOTE: ok:true means the lock re-confirmed after @@ -114,6 +119,7 @@ export function relockAndReconfirm( : DEFAULT_MAX_ATTEMPTS; const settleMs = opts.settleMs !== undefined ? opts.settleMs : resolveSettleMs(); const sleep = opts.sleep ?? sleepMs; + const confirm = opts.confirm ?? lock; let lastError = "Config re-lock did not re-confirm after settle window"; @@ -135,7 +141,7 @@ export function relockAndReconfirm( sleep(settleMs); try { - const confirmed = lock(); + const confirmed = confirm(); return { ok: true, attempts: attempt, lastResult: confirmed }; } catch (error: unknown) { // The reconciler reverted perms during the settle window. Retry the From 369bfa323cd0106864779001f76639a692698f10 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 28 Aug 2026 00:19:26 -0700 Subject: [PATCH 29/44] ci: retrigger PR review advisor Signed-off-by: Carlos Villela From 15141cdaf7a0d77092ce4cd03e269716292b5042 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:18:38 -0700 Subject: [PATCH 30/44] fix(shields): use read-only Hermes timer confirmation Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- src/lib/shields/index.ts | 2 ++ src/lib/shields/timer.test.ts | 61 +++++++++++++++++++++++++++++------ src/lib/shields/timer.ts | 28 ++++++++++++---- 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 80745f51916..019d4eea9f4 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -6985,6 +6985,7 @@ export { DEFAULT_TIMEOUT_SECONDS, deriveShieldsMode, getShieldsPosture, + hermesProviderLockConfirmation, inspectMutableConfigPerms, isShieldsDown, killTimer, @@ -6994,6 +6995,7 @@ export { prepareAutoRestoreTransitionTakeover, repairMutableConfigPerms, resolvePersistedAutoRestoreTarget, + resolveHermesShieldsProtocol, restoreLockedStateDirStartupAccess, shieldsDown, shieldsStatus, diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index 80db4419816..0ac03e793b7 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -21,9 +21,11 @@ const shieldsIndexMock = vi.hoisted(() => ({ } => ({ status: 0 }), ), completeAutoRestoreTransition: vi.fn(() => true), + hermesProviderLockConfirmation: vi.fn() as unknown, lockAgentConfig: vi.fn() as unknown, prepareAutoRestoreTransitionTakeover: vi.fn(), resolvePersistedAutoRestoreTarget: vi.fn() as unknown, + resolveHermesShieldsProtocol: vi.fn() as unknown, })); const PROCESS_TOKEN = "a".repeat(32); @@ -36,6 +38,9 @@ interface TimerTestOptions { vi.mock("./index", () => ({ applyShieldsPolicySnapshot: shieldsIndexMock.applyShieldsPolicySnapshot, completeAutoRestoreTransition: shieldsIndexMock.completeAutoRestoreTransition, + get hermesProviderLockConfirmation() { + return shieldsIndexMock.hermesProviderLockConfirmation; + }, get lockAgentConfig() { return shieldsIndexMock.lockAgentConfig; }, @@ -43,6 +48,9 @@ vi.mock("./index", () => ({ get resolvePersistedAutoRestoreTarget() { return shieldsIndexMock.resolvePersistedAutoRestoreTarget; }, + get resolveHermesShieldsProtocol() { + return shieldsIndexMock.resolveHermesShieldsProtocol; + }, })); describe("shields timer authorization", () => { @@ -52,7 +60,9 @@ describe("shields timer authorization", () => { tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-timer-")); vi.stubEnv("HOME", tmpHome); shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementation(() => ({ status: 0 })); + shieldsIndexMock.hermesProviderLockConfirmation = vi.fn(() => undefined); shieldsIndexMock.lockAgentConfig = vi.fn(); + shieldsIndexMock.resolveHermesShieldsProtocol = vi.fn(() => "sealed-plan-v1"); shieldsIndexMock.resolvePersistedAutoRestoreTarget = vi.fn( ( _sandboxName: string, @@ -1079,14 +1089,16 @@ describe("shields timer authorization", () => { expect(fs.existsSync(markerPath)).toBe(true); }); - it("persists chattrApplied and fileHashes from the auto-restore lock result", async () => { + it("persists the Hermes lock result after one runtime provider state mutation and read-only confirmation (#10155)", async () => { const stateDir = path.join(tmpHome, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); const sandboxName = "alpha"; - const configPath = "/sandbox/.openclaw/openclaw.json"; - const configDir = "/sandbox/.openclaw"; + const agentName = "hermes"; + const configPath = "/sandbox/.hermes/config.yaml"; + const configDir = "/sandbox/.hermes"; const sensitiveHashPath = `${configDir}/.config-hash`; + const sensitiveEnvPath = `${configDir}/.env`; const snapshotPath = path.join(stateDir, "snapshot.yaml"); const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); @@ -1101,20 +1113,32 @@ describe("shields timer authorization", () => { snapshotPath, restoreAt: restoreAtIso, processToken: PROCESS_TOKEN, + agentName, }), ); const sealedHashes = { [configPath]: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", [sensitiveHashPath]: "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + [sensitiveEnvPath]: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", }; const lockMock = vi.fn(() => ({ + chattrApplied: false, + fileHashes: { [configPath]: "0".repeat(64) }, + })); + const confirmMock = vi.fn(() => ({ chattrApplied: true, fileHashes: sealedHashes, })); const indexModule = await import("./index"); (indexModule.lockAgentConfig as ReturnType).mockImplementation(lockMock); + ( + indexModule.resolveHermesShieldsProtocol as ReturnType + ).mockReturnValue("provider-state-mutation-v2"); + ( + indexModule.hermesProviderLockConfirmation as ReturnType + ).mockReturnValue(confirmMock); const timer = await import("./timer"); const args = timer.parseTimerArgs([ @@ -1124,6 +1148,10 @@ describe("shields timer authorization", () => { configPath, configDir, PROCESS_TOKEN, + "0", + "", + "", + agentName, ]); expect(args).not.toBeNull(); @@ -1137,10 +1165,11 @@ describe("shields timer authorization", () => { snapshotPath, { deadlineAuthoritative: true, transitionProcessToken: PROCESS_TOKEN }, ); - // #4663: relockAndReconfirm applies then re-confirms after the settle - // window (0ms under test), so lockAgentConfig is invoked twice for a clean - // lock. - expect(lockMock).toHaveBeenCalledTimes(2); + expect(lockMock).toHaveBeenCalledTimes(1); + expect(confirmMock).toHaveBeenCalledTimes(1); + expect(lockMock.mock.invocationCallOrder[0]).toBeLessThan( + confirmMock.mock.invocationCallOrder[0]!, + ); expect(updatedState.shieldsDown).toBe(false); expect(updatedState.chattrApplied).toBe(true); expect(updatedState.fileHashes).toEqual(sealedHashes); @@ -1214,8 +1243,22 @@ describe("shields timer authorization", () => { ); expect(exitCode).toBe(0); expect(lockMock).toHaveBeenCalledTimes(2); - expect(lockMock).toHaveBeenNthCalledWith(1, sandboxName, fallbackTarget, false, false); - expect(lockMock).toHaveBeenNthCalledWith(2, sandboxName, fallbackTarget, false, false); + expect(lockMock).toHaveBeenNthCalledWith( + 1, + sandboxName, + fallbackTarget, + false, + false, + "sealed-plan-v1", + ); + expect(lockMock).toHaveBeenNthCalledWith( + 2, + sandboxName, + fallbackTarget, + false, + false, + "sealed-plan-v1", + ); expect(fs.existsSync(markerPath)).toBe(false); }); diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index e596e0fda73..3c761e40f25 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -460,13 +460,27 @@ async function runRestoreTimerWithBudget( // has settled, re-applying if it drifted. This narrows (does not // close) the revert window; fail closed (leave shields DOWN + audit) // when the lock will not re-confirm within the retry budget. - const relock = relockAndReconfirm(() => - lockAgentConfig( - args.sandboxName, - lockTarget, - false, - args.allowLegacyHermesProtocol, - ), + const protocol = shields.resolveHermesShieldsProtocol( + args.sandboxName, + lockTarget, + args.allowLegacyHermesProtocol, + ); + const relock = relockAndReconfirm( + () => + lockAgentConfig( + args.sandboxName, + lockTarget, + false, + args.allowLegacyHermesProtocol, + protocol, + ), + { + confirm: shields.hermesProviderLockConfirmation( + args.sandboxName, + lockTarget, + protocol, + ), + }, ); if (relock.ok && relock.lastResult) { lockedChattr = relock.lastResult.chattrApplied; From 30e78c8204ba5e2819a605db1c07edc1fd734467 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:09:47 -0700 Subject: [PATCH 31/44] fix(shields): confirm fresh Hermes lockdown Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- src/lib/shields/index.ts | 25 +- .../hermes-shields-up-confirmation.test.ts | 108 ++++++ ...me-state-mutation-hermes-publisher.test.ts | 315 ++++++++++++------ 3 files changed, 341 insertions(+), 107 deletions(-) create mode 100644 test/state/hermes-shields-up-confirmation.test.ts diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 019d4eea9f4..c8dd62c258a 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -6537,22 +6537,25 @@ function shieldsUpWithoutHostLock(sandboxName: string, opts: ShieldsUpOpts = {}) // Each operation runs independently and the result is verified. // If verification fails, config remains unlocked — we do not lie about state. console.log(` Locking ${target.agentName} config (${target.configPath})...`); - let lockResult: { chattrApplied: boolean; fileHashes: { [path: string]: string } }; - try { - lockResult = lockAgentConfig( - sandboxName, - target, - false, - opts.allowLegacyHermesProtocol === true, - protocol, - ); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const relock = relockAndReconfirm( + () => + lockAgentConfig( + sandboxName, + target, + false, + opts.allowLegacyHermesProtocol === true, + protocol, + ), + { confirm: hermesProviderLockConfirmation(sandboxName, target, protocol) }, + ); + if (!relock.ok || !relock.lastResult) { + const message = relock.error ?? "Config lock did not re-confirm after settle window"; console.error(` ERROR: ${message}`); console.error(" Config remains unlocked — manual intervention required."); printManualRelockRecoveryHint(sandboxName); return failShieldsCommand(message, opts.throwOnError); } + const lockResult = relock.lastResult; saveShieldsState(sandboxName, { chattrApplied: lockResult.chattrApplied, fileHashes: lockResult.fileHashes, diff --git a/test/state/hermes-shields-up-confirmation.test.ts b/test/state/hermes-shields-up-confirmation.test.ts new file mode 100644 index 00000000000..f4bf6172356 --- /dev/null +++ b/test/state/hermes-shields-up-confirmation.test.ts @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + createHermesShieldsProviderConsumerHarness, + hermesProviderConsumerSandbox as sandbox, +} from "../helpers/hermes-shields-provider-consumer-harness"; + +const requireSource = createRequire( + path.join(import.meta.dirname, "../../src/lib/shields/index.js"), +); + +describe("managed Hermes shields-up confirmation", () => { + let harness: ReturnType; + + beforeEach(() => { + harness = createHermesShieldsProviderConsumerHarness(requireSource); + }); + + afterEach(() => { + harness.cleanup(); + }); + + it("commits Shields up after one runtime provider state mutation and read-only confirmation (#10155)", () => { + const statePaths = requireSource( + "../state/paths.js", + ) as typeof import("../../src/lib/state/paths"); + const statePath = path.join( + statePaths.resolveNemoclawStateDir(), + "shields-" + sandbox.name + ".json", + ); + const events: string[] = []; + let verification = 0; + harness.transitionSpy.mockImplementation(() => { + expect(fs.existsSync(statePath)).toBe(false); + events.push("provider-mutation"); + return { fence: {}, proof: {} }; + }); + harness.verifyLockSpy.mockImplementation(() => { + expect(fs.existsSync(statePath)).toBe(false); + events.push(++verification === 1 ? "apply-verification" : "read-only-confirmation"); + return { issues: [] }; + }); + harness.auditSpy.mockImplementation(() => { + expect(JSON.parse(fs.readFileSync(statePath, "utf8"))).toMatchObject({ + shieldsDown: false, + }); + events.push("up-commit"); + }); + + harness.shields.shieldsUp(sandbox.name, { throwOnError: true }); + + expect(events).toEqual([ + "provider-mutation", + "apply-verification", + "read-only-confirmation", + "up-commit", + ]); + expect(harness.transitionSpy).toHaveBeenCalledOnce(); + expect(harness.transitionSpy).toHaveBeenCalledWith( + expect.objectContaining({ target: "locked", rollback: "mutable" }), + ); + expect(harness.verifyLockSpy).toHaveBeenCalledTimes(2); + expect(JSON.parse(fs.readFileSync(statePath, "utf8"))).toMatchObject({ + shieldsDown: false, + fileHashes: { + "/sandbox/.hermes/config.yaml": "c".repeat(64), + "/sandbox/.hermes/.env": "c".repeat(64), + "/sandbox/.hermes/.config-hash": "c".repeat(64), + }, + }); + }); + + it("does not commit Shields up when read-only confirmation finds persistent drift (#10155)", () => { + const statePaths = requireSource( + "../state/paths.js", + ) as typeof import("../../src/lib/state/paths"); + const statePath = path.join( + statePaths.resolveNemoclawStateDir(), + "shields-" + sandbox.name + ".json", + ); + let verification = 0; + harness.verifyLockSpy.mockImplementation(() => { + expect(fs.existsSync(statePath)).toBe(false); + verification += 1; + return { + issues: verification % 2 === 0 ? ["settled provider lock drift"] : [], + }; + }); + + expect(() => harness.shields.shieldsUp(sandbox.name, { throwOnError: true })).toThrow( + /settled provider lock drift/u, + ); + + expect(harness.transitionSpy).toHaveBeenCalledTimes(3); + expect(harness.transitionSpy).toHaveBeenCalledWith( + expect.objectContaining({ target: "locked", rollback: "mutable" }), + ); + expect(harness.verifyLockSpy).toHaveBeenCalledTimes(6); + expect(fs.existsSync(statePath)).toBe(false); + expect(harness.auditSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index 0576ea12c2c..0735bd5dc88 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -9,10 +9,8 @@ import { describe, expect, it } from "vitest"; const ROOT = path.join(import.meta.dirname, "../.."); const PUBLISHER = path.join(ROOT, "scripts", "runtime_state_mutation_hermes_publisher.py"); -const CAPABILITY = path.join(ROOT, "agents", "hermes", "runtime-state-mutation-publisher-v1.json"); const STATE_PLAN = path.join(ROOT, "agents", "hermes", "state-lock-plan.json"); const START = path.join(ROOT, "agents", "hermes", "start.sh"); -const STARTUP_GATE = path.join(ROOT, "scripts", "runtime-state-mutation-startup-gate.py"); const HARNESS = String.raw` import hashlib @@ -329,14 +327,137 @@ function runHarness(): Record { return JSON.parse(result.stdout) as Record; } -function shellFunction(source: string, name: string, nextName: string): string { - const start = source.indexOf(`${name}() {`); - const end = source.indexOf(`\n${nextName}() {`, start); - expect(start, `Expected ${name} in agents/hermes/start.sh`).toBeGreaterThanOrEqual(0); - expect(end, `Expected ${nextName} after ${name} in agents/hermes/start.sh`).toBeGreaterThan( - start, +type EntrypointGateFixture = { + acknowledgePath: string; + gatePath: string; + harnessPath: string; + releasePath: string; + tracePath: string; +}; + +function createEntrypointGateFixture(temporary: string): EntrypointGateFixture { + const fixture = { + acknowledgePath: path.join(temporary, "release-ack.json"), + gatePath: path.join(temporary, "gate.sh"), + harnessPath: path.join(temporary, "entrypoint-harness.sh"), + releasePath: path.join(temporary, "release.json"), + tracePath: path.join(temporary, "trace.log"), + }; + fs.writeFileSync( + fixture.gatePath, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'action=""', + 'for argument in "$@"; do action="$argument"; done', + 'case "$action" in', + " admit)", + ' printf "admit\\n" >> "$NEMOCLAW_TEST_TRACE"', + ' if [ "$NEMOCLAW_TEST_GATE_MODE" = "deny" ]; then exit 1; fi', + " exit 10", + " ;;", + " checkpoint)", + ' printf "checkpoint:%s\\n" "$PPID" >> "$NEMOCLAW_TEST_TRACE"', + " exit 11", + " ;;", + " resume)", + ' if [ ! -f "$NEMOCLAW_TEST_RELEASE" ]; then exit 1; fi', + ' printf "resume:%s\\n" "$PPID" >> "$NEMOCLAW_TEST_TRACE"', + " ;;", + " acknowledge)", + ' temporary_ack="$NEMOCLAW_TEST_ACKNOWLEDGE.$$.tmp"', + ' printf \'{"pid":%s}\\n\' "$$" > "$temporary_ack"', + ' chmod 600 "$temporary_ack"', + ' mv "$temporary_ack" "$NEMOCLAW_TEST_ACKNOWLEDGE"', + ' printf "acknowledge:%s\\n" "$$" >> "$NEMOCLAW_TEST_TRACE"', + ' kill -STOP "$$"', + ' printf "acknowledged:%s\\n" "$$" >> "$NEMOCLAW_TEST_TRACE"', + " ;;", + " *) exit 1 ;;", + "esac", + ].join("\n"), + { mode: 0o700 }, + ); + fs.writeFileSync( + fixture.harnessPath, + [ + "function [ {", + ' case "$1:$2:$3" in', + ' "!:-x:/opt/hermes/.venv/bin/python3"|"!:-f:/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py"|"!:-x:/usr/bin/setpriv") return 1 ;;', + " esac", + ' builtin [ "$@"', + "}", + "function /opt/hermes/.venv/bin/python3 {", + ' "$NEMOCLAW_TEST_GATE" "$@"', + "}", + "function /usr/bin/setpriv {", + ' while [ "$#" -gt 0 ]; do', + ' case "$1" in --) shift; break ;; *) shift ;; esac', + " done", + ' "$@"', + "}", + "nemoclaw_test_checkpoint_entry() {", + ' case "$BASH_COMMAND" in', + " _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER=*)", + " trap - DEBUG", + ' printf "checkpoint-call\\n" >> "$NEMOCLAW_TEST_TRACE"', + " if nemoclaw_runtime_state_mutation_checkpoint; then exit 0; fi", + ' status="$?"', + ' exit "$status"', + " ;;", + " esac", + "}", + "set -T", + "trap nemoclaw_test_checkpoint_entry DEBUG", + 'source "$NEMOCLAW_TEST_ENTRYPOINT"', + ].join("\n"), + { mode: 0o700 }, ); - return source.slice(start, end); + return fixture; +} + +function readEntrypointTrace(tracePath: string): string[] { + try { + const content = fs.readFileSync(tracePath, "utf8").trim(); + return content.length === 0 ? [] : content.split("\n").filter(Boolean); + } catch { + return []; + } +} + +function processState(pid: number): string { + const result = spawnSync("ps", ["-o", "state=", "-p", String(pid)], { + encoding: "utf8", + timeout: 1_000, + }); + return result.status === 0 ? result.stdout.trim() : ""; +} + +async function waitForCondition(description: string, predicate: () => boolean): Promise { + const deadline = Date.now() + 5_000; + while (!predicate() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(predicate(), "Timed out waiting for " + description).toBe(true); +} + +async function waitForStoppedProcess(pid: number, description: string): Promise { + await waitForCondition(description, () => processState(pid).startsWith("T")); +} + +function entrypointGateEnvironment( + fixture: EntrypointGateFixture, + gateMode: "allow" | "deny", +): NodeJS.ProcessEnv { + return { + ...process.env, + NEMOCLAW_TEST_ACKNOWLEDGE: fixture.acknowledgePath, + NEMOCLAW_TEST_ENTRYPOINT: START, + NEMOCLAW_TEST_GATE: fixture.gatePath, + NEMOCLAW_TEST_GATE_MODE: gateMode, + NEMOCLAW_TEST_RELEASE: fixture.releasePath, + NEMOCLAW_TEST_TRACE: fixture.tracePath, + }; } describe("Hermes runtime state mutation publisher", () => { @@ -405,98 +526,100 @@ describe("Hermes runtime state mutation publisher", () => { ]); }); - it("publishes the release acknowledgement from the Hermes startup process (#10155)", () => { - const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-release-ack-")); - try { - const nonce = "a".repeat(64); - const gate = path.join(temporary, "gate.py"); - const waited = path.join(temporary, "controller-wait-finished"); - fs.writeFileSync( - gate, - `import json\nimport os\nimport sys\n\nexpected = int(os.environ["NEMOCLAW_TEST_EXPECTED_START_PID"])\nif os.getppid() != expected:\n raise SystemExit("gate-start-mismatch")\nnonce = os.environ["NEMOCLAW_TEST_NONCE"]\nroot = os.environ["NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT"]\ndirectory = os.path.join(root, nonce)\nos.mkdir(directory, 0o700)\nfinal = os.path.join(directory, "release-ack.json")\nwith open(final, "x", encoding="utf-8") as stream:\n json.dump({"nonce": nonce}, stream, separators=(",", ":"))\nos.chmod(final, 0o600)\nprint(nonce)\n`, - { mode: 0o700 }, + it( + "runs the Hermes entrypoint checkpoint, release acknowledgement, and parent resume in order (#10155)", + { timeout: 15_000 }, + async () => { + const temporary = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-hermes-entrypoint-checkpoint-"), ); - const start = fs.readFileSync(START, "utf8"); - const acknowledge = shellFunction( - start, - "nemoclaw_runtime_state_mutation_acknowledge_release", - "nemoclaw_runtime_state_mutation_retry_exec", + const fixture = createEntrypointGateFixture(temporary); + const child = spawn("bash", [fixture.harnessPath], { + env: entrypointGateEnvironment(fixture, "allow"), + stdio: ["ignore", "ignore", "pipe"], + }); + expect(child.pid).toBeTypeOf("number"); + const startPid = Number(child.pid); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += String(chunk); + }); + const completion = new Promise<{ signal: NodeJS.Signals | null; status: number | null }>( + (resolve) => { + child.once("exit", (status, signal) => resolve({ signal, status })); + }, ); - const controllerWait = - 'import os,sys,time\npath=sys.argv[1]\ndeadline=time.monotonic()+5\nwhile not os.path.isfile(path):\n if time.monotonic() >= deadline: raise SystemExit("release-ack-timeout")\n time.sleep(0.01)\nopen(sys.argv[2], "x").close()'; - const script = `${acknowledge} -export NEMOCLAW_TEST_EXPECTED_START_PID="$$" -"$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I -c "$NEMOCLAW_TEST_CONTROLLER_WAIT" \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT/$NEMOCLAW_TEST_NONCE/release-ack.json" \ - "$NEMOCLAW_TEST_WAITED" & -controller_pid=$! -nemoclaw_runtime_state_mutation_acknowledge_release -wait "$controller_pid" -`; - const result = spawnSync("bash", ["-c", script], { + + try { + await waitForCondition("Hermes entrypoint checkpoint", () => + readEntrypointTrace(fixture.tracePath).some((line) => line.startsWith("checkpoint:")), + ); + await waitForStoppedProcess(startPid, "Hermes entrypoint candidate stop"); + fs.writeFileSync(fixture.releasePath, '{"released":true}\n', { mode: 0o600 }); + expect(child.kill("SIGCONT")).toBe(true); + + await waitForCondition("release acknowledgement publication", () => + readEntrypointTrace(fixture.tracePath).some((line) => line.startsWith("acknowledge:")), + ); + const acknowledgeLine = readEntrypointTrace(fixture.tracePath).find((line) => + line.startsWith("acknowledge:"), + ); + const acknowledgePid = Number(acknowledgeLine?.split(":")[1]); + expect(Number.isSafeInteger(acknowledgePid)).toBe(true); + expect(acknowledgePid).toBeGreaterThan(1); + await waitForStoppedProcess( + acknowledgePid, + "release acknowledgement child stop after publication", + ); + process.kill(acknowledgePid, "SIGCONT"); + + await waitForCondition("release acknowledgement completion", () => + readEntrypointTrace(fixture.tracePath).includes("acknowledged:" + String(acknowledgePid)), + ); + await waitForStoppedProcess( + startPid, + "Hermes entrypoint parent stop after acknowledged child completion", + ); + expect(child.kill("SIGCONT")).toBe(true); + + const result = await completion; + expect(result.status, stderr).toBe(0); + expect(result.signal).toBeNull(); + expect(JSON.parse(fs.readFileSync(fixture.acknowledgePath, "utf8"))).toEqual({ + pid: acknowledgePid, + }); + expect(readEntrypointTrace(fixture.tracePath)).toEqual([ + "admit", + "checkpoint-call", + "checkpoint:" + String(startPid), + "resume:" + String(startPid), + "acknowledge:" + String(acknowledgePid), + "acknowledged:" + String(acknowledgePid), + ]); + } finally { + child.kill("SIGKILL"); + await completion; + fs.rmSync(temporary, { force: true, recursive: true }); + } + }, + ); + + it("stops the actual Hermes entrypoint before startup when the durable gate denies admission (#10155)", () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-entrypoint-denied-")); + const fixture = createEntrypointGateFixture(temporary); + try { + const result = spawnSync("bash", [fixture.harnessPath], { encoding: "utf8", - timeout: 10_000, - env: { - ...process.env, - NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON: "python3", - NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER: gate, - NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV: "setpriv", - NEMOCLAW_RUNTIME_STATE_MUTATION_HANDOFF_ROOT: temporary, - NEMOCLAW_TEST_CONTROLLER_WAIT: controllerWait, - NEMOCLAW_TEST_NONCE: nonce, - NEMOCLAW_TEST_WAITED: waited, - }, + env: entrypointGateEnvironment(fixture, "deny"), + timeout: 5_000, }); - expect(result.status, result.stderr).toBe(0); - expect(fs.readFileSync(path.join(temporary, nonce, "release-ack.json"), "utf8")).toBe( - `{"nonce":"${nonce}"}`, - ); - expect(fs.existsSync(waited)).toBe(true); - expect(acknowledge).not.toContain("mktemp"); - expect(acknowledge).not.toContain("mv"); - const checkpoint = start.indexOf( - "if nemoclaw_runtime_state_mutation_acknowledge_release; then", - ); - const parentStop = start.indexOf('kill -STOP "$$"', checkpoint); - const acknowledgedReturn = start.indexOf("return 0", parentStop); - expect(checkpoint).toBeGreaterThan(0); - expect(parentStop).toBeGreaterThan(checkpoint); - expect(acknowledgedReturn).toBeGreaterThan(parentStop); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Runtime state mutation startup gate failed"); + expect(readEntrypointTrace(fixture.tracePath)).toEqual(["admit"]); + expect(fs.existsSync(fixture.acknowledgePath)).toBe(false); } finally { fs.rmSync(temporary, { force: true, recursive: true }); } }); - - it("publishes one exact image capability and checks the durable root gate before startup code", () => { - expect(fs.readFileSync(CAPABILITY, "utf8")).toBe( - '{"schemaVersion":1,"protocol":"nemoclaw-runtime-state-mutation-publisher-v1","agent":"hermes","providerId":"docker","stateRoot":"/sandbox/.hermes","planSchemaVersion":2,"entrypoint":"/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py"}\n', - ); - const start = fs.readFileSync(START, "utf8"); - const gate = start.indexOf( - 'NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER="/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py"', - ); - expect(gate).toBeGreaterThan(0); - expect(gate).toBeLessThan(start.indexOf("# managed-entrypoint-env-wrapper begin")); - expect(gate).toBeLessThan(start.indexOf('source "$_SANDBOX_INIT"')); - expect(gate).toBeLessThan(start.indexOf('migrate_legacy_layout "/sandbox/.hermes"')); - expect(start).not.toContain("/sandbox/.nemoclaw/runtime-state-mutation-hold-v1.json"); - expect(start).toContain( - 'NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON="/opt/hermes/.venv/bin/python3"', - ); - expect(start).toContain('NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV="/usr/bin/setpriv"'); - expect(start).toContain("--reuid=sandbox --regid=sandbox --init-groups --"); - expect(start).toContain("nemoclaw_runtime_state_mutation_checkpoint || return 1"); - expect(start).toContain("trap nemoclaw_runtime_state_mutation_retry_exec USR2"); - expect(start).toContain("exec /usr/local/bin/nemoclaw-start"); - expect(start).toContain("nemoclaw_runtime_state_mutation_gate resume"); - - const startupGate = fs.readFileSync(STARTUP_GATE, "utf8"); - expect(startupGate).toContain('DURABLE_DIRECTORY = "/var/lib/nemoclaw/runtime-state-mutation"'); - expect(startupGate).toContain('"permitted": 10'); - expect(startupGate).toContain('"activation-ready": 11'); - expect(startupGate).toContain('"retry": 12'); - expect(fs.readFileSync(PUBLISHER, "utf8")).toContain( - 'PYTHON_PATH = "/opt/hermes/.venv/bin/python3"', - ); - }); }); From 6e66abad83600ee17114ca2b45d29ce8824f0c0f Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:46:20 -0700 Subject: [PATCH 32/44] fix(runtime): retain Hermes transport recovery Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- agents/hermes/start.sh | 14 +- scripts/runtime-state-mutation-control.py | 2 - .../docker-state-mutation.test.ts | 146 ++++++++++-------- .../runtime-provider/docker-state-mutation.ts | 115 ++++++++++++-- .../persisted-engine-lifecycle.ts | 13 +- test/helpers/docker-state-mutation-harness.ts | 44 ++++++ ...me-state-mutation-hermes-publisher.test.ts | 61 ++++---- ...runtime-state-mutation-release-ack.test.ts | 7 +- 8 files changed, 284 insertions(+), 118 deletions(-) diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 5ee08d18a05..a54367c6330 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -52,18 +52,6 @@ nemoclaw_runtime_state_mutation_gate() { "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" "$action" >/dev/null } -nemoclaw_runtime_state_mutation_acknowledge_release() { - if [ "$EUID" -eq 0 ]; then - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_SETPRIV" \ - --reuid=sandbox --regid=sandbox --init-groups -- \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >/dev/null - return - fi - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_PYTHON" -I \ - "$NEMOCLAW_RUNTIME_STATE_MUTATION_GATE_HELPER" acknowledge >/dev/null -} - nemoclaw_runtime_state_mutation_retry_exec() { local status if nemoclaw_runtime_state_mutation_gate restart; then @@ -118,7 +106,7 @@ nemoclaw_runtime_state_mutation_checkpoint() { fi kill -STOP "$$" if nemoclaw_runtime_state_mutation_gate resume; then - if nemoclaw_runtime_state_mutation_acknowledge_release; then + if nemoclaw_runtime_state_mutation_gate acknowledge; then # The acknowledgement helper stops itself after publishing. The root # controller resumes that exact child, then this parent stops only after # Bash has reaped it and observed success. diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index c58a64a363b..e85eb9c8fda 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -94,7 +94,6 @@ STARTUP_CANDIDATE_NAME = "startup-complete.json" STARTUP_RETRY_ACK_NAME = "retry-ack.json" STARTUP_RELEASE_ACK_NAME = "release-ack.json" -STARTUP_RELEASE_ACK_PENDING_NAME = ".release-ack.json.pending" RELEASED_RECEIPT_NAME = "released.json" PUBLISHER_MODULE_PATH = ( "/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py" @@ -2230,7 +2229,6 @@ def _cleanup_startup_candidate_directory(marker: dict[str, object]) -> None: STARTUP_CANDIDATE_NAME, STARTUP_RETRY_ACK_NAME, STARTUP_RELEASE_ACK_NAME, - STARTUP_RELEASE_ACK_PENDING_NAME, ) and not name.startswith(f".{STARTUP_CANDIDATE_NAME}.") and not name.startswith(f".{STARTUP_RETRY_ACK_NAME}.") diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 293c56a612a..a95ccc1ba49 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -26,17 +26,11 @@ import { import { createDockerOperationAuthority } from "./docker-operation-authority"; import { DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS, - DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_BOOTSTRAP, DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE, createDockerStateMutationOwner, createDockerStateMutationSurface, } from "./docker-state-mutation"; -const RUNTIME_STATE_MUTATION_CONTROLLER = path.join( - import.meta.dirname, - "../../../../scripts/runtime-state-mutation-control.py", -); - function ownerThatStopsAfterPrepare(runtime: ReturnType) { const acquireMutationExecution = vi.fn(() => { throw new Error("injected controller exit before helper invocation"); @@ -59,6 +53,18 @@ function ownerThatStopsAfterPrepare(runtime: ReturnType) { }; } +function finalizeReleasedProviderWithLiveTransport( + runtime: ReturnType, + transactionId: string, + resultSha256: string, +): void { + const lease = runtime.lifecycleStore.acquireMutationExecution(transactionId); + runtime.lifecycleStore.complete(lease, resultSha256); + runtime.releaseProviderWithoutTransportCleanup(); + runtime.lifecycleStore.finalizeStateMutationRelease(lease, resultSha256); + runtime.lifecycleStore.releaseMutationExecution(lease); +} + async function waitForPath(filePath: string, timeoutMs: number): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -159,61 +165,6 @@ except subprocess.TimeoutExpired: ).toBe("helper-timeout"); }); - it("keeps activation transport alive beyond both controller readiness windows", () => { - const probe = String.raw` -import importlib.util -import json -import sys - -spec = importlib.util.spec_from_file_location("runtime_state_control", sys.argv[1]) -control = importlib.util.module_from_spec(spec) -sys.modules[spec.name] = control -spec.loader.exec_module(control) -print(json.dumps({ - "activationSeconds": control.ACTIVATION_SECONDS, - "processStateSeconds": control.PROCESS_STATE_SECONDS, -}, separators=(",", ":"))) -`; - const timing = JSON.parse( - execFileSync("python3", ["-I", "-c", probe, RUNTIME_STATE_MUTATION_CONTROLLER], { - encoding: "utf8", - timeout: 5_000, - }), - ) as { activationSeconds: number; processStateSeconds: number }; - - expect(DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS / 1000).toBeGreaterThan( - timing.activationSeconds * 2 + timing.processStateSeconds * 2, - ); - }); - - it("binds the guardian recovery lineage to the exact embedded broker command", () => { - const probe = String.raw` -import importlib.util -import json -import sys - -spec = importlib.util.spec_from_file_location("runtime_state_control", sys.argv[1]) -control = importlib.util.module_from_spec(spec) -sys.modules[spec.name] = control -spec.loader.exec_module(control) -print(json.dumps({ - "bootstrap": control.TRANSPORT_BROKER_BOOTSTRAP.decode("ascii"), - "helper": control.TRANSPORT_BROKER_HELPER_PATH.decode("ascii"), -}, separators=(",", ":"))) -`; - const binding = JSON.parse( - execFileSync("python3", ["-I", "-c", probe, RUNTIME_STATE_MUTATION_CONTROLLER], { - encoding: "utf8", - timeout: 5_000, - }), - ) as { bootstrap: string; helper: string }; - - expect(binding).toEqual({ - bootstrap: DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_BOOTSTRAP, - helper: "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", - }); - }); - it("keeps the signal replay inside one activation broker deadline (#10155)", () => { const definitionsEnd = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.indexOf( "\nhelper = sys.argv[1]\n", @@ -398,6 +349,56 @@ print(json.dumps(timeouts)) expect(exclusionCalls).toEqual([["alpha", "Docker runtime-provider state mutation recovery"]]); }); + it("recovers transport left by a finalized provider release (#10155)", () => { + const runtime = harness({ failReleaseCleanupInspectionOnce: true }); + const surface = createDockerStateMutationSurface({ + capture: runtime.capture, + resolveStateDir: () => runtime.root, + }); + const fence = surface.acquire({ ...runtime.context, plan: plan() }); + surface.activate(runtime.context, fence); + const completedLedgerSha256 = "e".repeat(64); + finalizeReleasedProviderWithLiveTransport(runtime, fence.transactionId, completedLedgerSha256); + + expect(runtime.lifecycleStore.listUnfinished()).toEqual([]); + expect(runtime.transportBrokerSessionExists(fence.transactionId)).toBe(true); + + expect(() => surface.recover(runtime.context)).toThrow( + "root helper transport release cleanup verification failed", + ); + expect(runtime.lifecycleStore.load(fence.transactionId)).toMatchObject({ + phase: "completed", + resultSha256: completedLedgerSha256, + }); + expect(runtime.transportBrokerSessionExists(fence.transactionId)).toBe(true); + + expect(surface.recover(runtime.context)).toBeNull(); + expect(runtime.transportBrokerSessionExists(fence.transactionId)).toBe(false); + expect(runtime.lifecycleStore.isRetired(fence.transactionId, completedLedgerSha256)).toBe(true); + }); + + it("cleans finalized provider transport before the next acquire (#10155)", () => { + const runtime = harness(); + const surface = createDockerStateMutationSurface({ + capture: runtime.capture, + resolveStateDir: () => runtime.root, + }); + const first = surface.acquire({ ...runtime.context, plan: plan() }); + surface.activate(runtime.context, first); + const completedLedgerSha256 = "e".repeat(64); + finalizeReleasedProviderWithLiveTransport(runtime, first.transactionId, completedLedgerSha256); + + const second = surface.acquire({ ...runtime.context, plan: plan() }); + + expect(second.transactionId).not.toBe(first.transactionId); + expect(runtime.transportBrokerSessionExists(first.transactionId)).toBe(false); + expect(runtime.transportBrokerSessionExists(second.transactionId)).toBe(true); + expect(runtime.lifecycleStore.isRetired(first.transactionId, completedLedgerSha256)).toBe(true); + expect(runtime.lifecycleStore.listUnfinished()).toMatchObject([ + { phase: "fence-established", transactionId: second.transactionId }, + ]); + }); + it("preserves the isolated Vitest state root from the operation environment", () => { const runtime = harness(); const surface = createDockerStateMutationSurface({ capture: runtime.capture }); @@ -752,6 +753,29 @@ describe("Docker state mutation owner", () => { expect(runtime.lifecycleStore.listUnfinished()).toEqual([]); }); + it("retains release recovery until Docker broker cleanup completes (#10155)", () => { + const runtime = harness({ failReleaseCleanupProbeOnce: true }); + const fence = runtime.owner.acquire({ ...runtime.context, plan: plan() }); + runtime.owner.rollback(runtime.context, fence); + const proof = runtime.owner.activate(runtime.context, fence); + const completedLedgerSha256 = "e".repeat(64); + + expect(() => + runtime.owner.release(runtime.context, fence, proof, completedLedgerSha256), + ).toThrow("root helper transport release cleanup remains incomplete"); + expect(runtime.transportBrokerActive()).toBe(true); + expect(runtime.transportBrokerSessionExists()).toBe(true); + expect(runtime.lifecycleStore.listUnfinished()[0]).toMatchObject({ + phase: "completed", + resultSha256: completedLedgerSha256, + }); + + expect(runtime.owner.recover(runtime.context)).toBeNull(); + expect(runtime.transportBrokerActive()).toBe(false); + expect(runtime.transportBrokerSessionExists()).toBe(false); + expect(runtime.lifecycleStore.listUnfinished()).toEqual([]); + }); + it("recovers a durable provider-release receipt without requiring the removed marker", () => { const runtime = harness(); const fence = runtime.owner.acquire({ ...runtime.context, plan: plan() }); diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index e06ca74402d..0db4d33a3ea 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -75,6 +75,16 @@ const HELPER_TRANSPORT_COMMAND_TIMEOUT_MS = 15_000; const HELPER_TRANSPORT_RESPONSE_ALLOWANCE_MS = 30_000; const HELPER_TRANSPORT_POLL_MS = 250; const HELPER_TRANSPORT_ROOT = "/run/nemoclaw/runtime-state-mutation"; +const HELPER_TRANSPORT_SESSION_PRESENT_STATUS = 75; +const HELPER_TRANSPORT_SESSION_INSPECTION_FAILED_STATUS = 76; +const HELPER_TRANSPORT_SESSION_ABSENT_SCRIPT = `import os,sys +try: + os.lstat(sys.argv[1]) +except FileNotFoundError: + raise SystemExit(0) +except Exception: + raise SystemExit(${HELPER_TRANSPORT_SESSION_INSPECTION_FAILED_STATUS}) +raise SystemExit(${HELPER_TRANSPORT_SESSION_PRESENT_STATUS})`; export const DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_BOOTSTRAP = "import base64,sys,zlib;source=zlib.decompress(base64.b64decode(sys.argv.pop(1)),-15);exec(compile(source,'','exec'))"; const MAX_HELPER_TRANSPORT_BYTES = 128 * 1024; @@ -1366,6 +1376,27 @@ function helperTransportBrokerCommand( }); } +function helperTransportSessionAbsentCommand( + runtimeId: string, + transactionId: string, +): PersistedEngineLifecycleExactCommand { + return Object.freeze({ + args: Object.freeze([ + "container", + "exec", + "--user", + "root", + runtimeId, + HELPER_PYTHON_PATH, + "-I", + "-c", + HELPER_TRANSPORT_SESSION_ABSENT_SCRIPT, + helperTransportSessionPath(transactionId), + ]), + targetIndex: 4, + }); +} + function helperTransportCopyToCommand( runtimeId: string, hostPath: string, @@ -1547,7 +1578,22 @@ function finishReleasedHelperTransport( requireCurrentEngineAuthority(options, bindingSha256); return result; }; - if (!probeHelperTransport(capture, options, transactionId)) return; + const sessionAbsent = (): boolean => { + const result = capture( + helperTransportSessionAbsentCommand(options.runtimeId, transactionId), + HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, + ); + if (result.error || result.stderr.length !== 0) { + fail("root helper transport release cleanup verification failed"); + } + if (result.status === 0) return true; + if (result.status === HELPER_TRANSPORT_SESSION_PRESENT_STATUS) return false; + fail("root helper transport release cleanup verification failed"); + }; + if (sessionAbsent()) return; + if (!probeHelperTransport(capture, options, transactionId)) { + fail("root helper transport release cleanup remains incomplete"); + } withHelperTransportHostDirectory(options.hostTransportRoot, (temporary) => { const resumed = path.join(temporary, "resumed"); writePrivateTransportFile(resumed, Buffer.from(`${transactionId}\n`, "ascii")); @@ -1563,6 +1609,13 @@ function finishReleasedHelperTransport( "root helper transport release finalization", ); }); + const deadline = Date.now() + HELPER_TRANSPORT_COMMAND_TIMEOUT_MS; + while (!sessionAbsent()) { + if (Date.now() >= deadline) { + fail("root helper transport release cleanup remains incomplete"); + } + Atomics.wait(helperTransportPoll, 0, 0, HELPER_TRANSPORT_POLL_MS); + } } function parseHelperTransportResult( @@ -2581,8 +2634,8 @@ export function createContainerStateMutationOwner( proof, completedLedgerSha256, ); + finishReleasedHelperTransport(options, bindingSha256, record.transactionId); }); - finishReleasedHelperTransport(options, bindingSha256, record.transactionId); options.lifecycleStore.retire(record.transactionId, completedLedgerSha256); return; } @@ -2628,9 +2681,9 @@ export function createContainerStateMutationOwner( activatedProof, completedLedgerSha256, ); + finishReleasedHelperTransport(options, bindingSha256, record.transactionId); }, ); - finishReleasedHelperTransport(options, bindingSha256, record.transactionId); options.lifecycleStore.retire(record.transactionId, completedLedgerSha256); }, @@ -2669,8 +2722,8 @@ export function createContainerStateMutationOwner( recoveredProof, completed.resultSha256 as string, ); + finishReleasedHelperTransport(options, bindingSha256, record.transactionId); }); - finishReleasedHelperTransport(options, bindingSha256, record.transactionId); options.lifecycleStore.retire(record.transactionId, record.resultSha256); return null; } @@ -2786,11 +2839,11 @@ function requireExistingSurfaceAuthority( ); } -function createSurfaceOwner( +function createSurfaceOwnerOptions( input: RuntimeProviderStateMutationContext, options: ContainerStateMutationSurfaceOptions, phase: "acquire" | "existing", -): ContainerStateMutationOwner { +): ContainerStateMutationOwnerOptions { requireSurfaceContext(input, options.providerId); const authority = options.createAuthority(input); @@ -2806,7 +2859,7 @@ function createSurfaceOwner( input.sandboxName, options.providerDisplayName, ); - return createContainerStateMutationOwner({ + return { providerId: options.providerId, providerDisplayName: options.providerDisplayName, engineOperation: options.engineOperation, @@ -2822,6 +2875,50 @@ function createSurfaceOwner( authority, engineAuthorityStore, lifecycleStore: createFilePersistedEngineLifecycleStore(stateDir), + }; +} + +function createSurfaceOwner( + input: RuntimeProviderStateMutationContext, + options: ContainerStateMutationSurfaceOptions, + phase: "acquire" | "existing", +): ContainerStateMutationOwner { + return createContainerStateMutationOwner(createSurfaceOwnerOptions(input, options, phase)); +} + +function retireReleasedSurfaceStateMutations( + input: RuntimeProviderStateMutationContext, + options: ContainerStateMutationSurfaceOptions, + lifecycleStore: PersistedEngineLifecycleStore, +): void { + let cleanup: + | { + readonly ownerOptions: ContainerStateMutationOwnerOptions; + readonly bindingSha256: string; + } + | undefined; + lifecycleStore.retireReleasedStateMutations(input.sandboxName, (record) => { + if (options.providerId !== DOCKER_PROVIDER_ID) return; + if (!cleanup) { + const ownerOptions = createSurfaceOwnerOptions(input, options, "existing"); + cleanup = { + ownerOptions, + bindingSha256: operationBindingSha256(ownerOptions.authority.engine), + }; + } + const targetRuntime = record.resources.find( + (resource) => resource.role === "target", + )?.runtimeId; + if (targetRuntime !== cleanup.ownerOptions.runtimeId) { + fail( + `released state mutation target does not match the exact labeled ${options.providerDisplayName} runtime`, + ); + } + finishReleasedHelperTransport( + cleanup.ownerOptions, + cleanup.bindingSha256, + record.transactionId, + ); }); } @@ -2834,7 +2931,7 @@ function recoverSurface( input.environment, ); const lifecycleStore = createFilePersistedEngineLifecycleStore(stateDir); - lifecycleStore.retireReleasedStateMutations(input.sandboxName); + retireReleasedSurfaceStateMutations(input, options, lifecycleStore); if (!hasActivePersistedEngineStateMutationTarget(lifecycleStore, input.sandboxName)) { return null; } @@ -2885,7 +2982,7 @@ export function createContainerStateMutationSurface( input.environment, ); const lifecycleStore = createFilePersistedEngineLifecycleStore(stateDir); - lifecycleStore.retireReleasedStateMutations(input.sandboxName); + retireReleasedSurfaceStateMutations(input, options, lifecycleStore); if (hasActivePersistedEngineStateMutationTarget(lifecycleStore, input.sandboxName)) { fail("OpenShell sandbox already has one unfinished state mutation"); } diff --git a/src/lib/onboard/runtime-provider/persisted-engine-lifecycle.ts b/src/lib/onboard/runtime-provider/persisted-engine-lifecycle.ts index 52c729f17fd..dcf5ccc35a8 100644 --- a/src/lib/onboard/runtime-provider/persisted-engine-lifecycle.ts +++ b/src/lib/onboard/runtime-provider/persisted-engine-lifecycle.ts @@ -130,8 +130,11 @@ export interface PersistedEngineLifecycleStore { ) => boolean; /** Retire only the exact completed receipt; a durable tombstone prevents ID reuse. */ readonly retire: (transactionId: string, resultSha256: string) => void; - /** Finish safe cleanup left after a released mutation lost its final response. */ - readonly retireReleasedStateMutations: (sandboxName: string) => void; + /** Finish provider cleanup before retiring a released mutation's durable receipt. */ + readonly retireReleasedStateMutations: ( + sandboxName: string, + beforeRetire: (record: PersistedEngineLifecycleRecord) => void, + ) => void; /** Match only an exact durable retirement receipt. */ readonly isRetired: (transactionId: string, resultSha256: string) => boolean; } @@ -1787,7 +1790,10 @@ export function createFilePersistedEngineLifecycleStore( retire(transactionId: string, resultSha256: string) { retireCompletedTransaction(root, transactionId, resultSha256); }, - retireReleasedStateMutations(sandboxName: string) { + retireReleasedStateMutations( + sandboxName: string, + beforeRetire: (record: PersistedEngineLifecycleRecord) => void, + ) { const exactSandboxName = exactName(sandboxName, "sandbox name"); requirePrivateDirectory(root); for (const entry of fs.readdirSync(root).sort()) { @@ -1807,6 +1813,7 @@ export function createFilePersistedEngineLifecycleStore( hasFinalizedStateMutationRelease(root, current) && !hasMatchingRuntimeTargetClaim(root, current) ) { + beforeRetire(current); retireCompletedTransaction(root, current.transactionId, current.resultSha256 as string); } } diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index b3e76cc3656..4a578ee255a 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -146,7 +146,9 @@ export interface DockerStateMutationHarnessOptions { readonly afterHelper?: (action: string, state: DockerStateMutationHarnessState) => void; readonly deferAcquireOnce?: boolean; readonly failAcquire?: boolean; + readonly failReleaseCleanupInspectionOnce?: boolean; readonly failReleaseOnce?: boolean; + readonly failReleaseCleanupProbeOnce?: boolean; readonly failResumeOnce?: boolean; readonly lifecycleGeneration?: string; readonly loseAcquireResponseOnce?: boolean; @@ -198,6 +200,8 @@ function createContainerStateMutationHarness( let acquireDeferralsRemaining = options.deferAcquireOnce ? 1 : 0; let lostAcquireResponsesRemaining = options.loseAcquireResponseOnce ? 1 : 0; let releaseFailuresRemaining = options.failReleaseOnce ? 1 : 0; + let releaseCleanupInspectionFailuresRemaining = options.failReleaseCleanupInspectionOnce ? 1 : 0; + let releaseCleanupProbeFailuresRemaining = options.failReleaseCleanupProbeOnce ? 1 : 0; let resumeFailuresRemaining = options.failResumeOnce ? 1 : 0; let lostReleaseResponsesRemaining = options.loseReleaseResponseOnce ? 1 : 0; let signalledHelpersRemaining = options.signalHelperOnce ? 1 : 0; @@ -210,6 +214,17 @@ function createContainerStateMutationHarness( let brokerTransactionId: string | null = null; const transportFiles = new Map(); + const inspectTransportSession = (session: string) => { + if (releaseCleanupInspectionFailuresRemaining > 0) { + releaseCleanupInspectionFailuresRemaining -= 1; + return { status: 76, stdout: "", stderr: "" }; + } + const exists = + brokerActive || + [...transportFiles.keys()].some((file) => file === session || file.startsWith(`${session}/`)); + return { status: exists ? 75 : 0, stdout: "", stderr: "" }; + }; + const acquireMarker = (request: Record) => { const candidate = { schemaVersion: 1, @@ -439,6 +454,14 @@ function createContainerStateMutationHarness( } if (source.startsWith(containerPrefix)) { const containerPath = source.slice(containerPrefix.length); + if ( + containerPath.endsWith("/ready") && + brokerReleased && + releaseCleanupProbeFailuresRemaining > 0 + ) { + releaseCleanupProbeFailuresRemaining -= 1; + return { status: 1, stdout: "", stderr: "transport readiness unavailable" }; + } if (containerPath.endsWith(".response") && responseCopyTimeoutsRemaining > 0) { responseCopyTimeoutsRemaining -= 1; return { @@ -460,6 +483,12 @@ function createContainerStateMutationHarness( if (command[0] !== "container" || command[1] !== "exec") { return { status: 1, stdout: "", stderr: "unexpected command" }; } + if ( + command[7] === "-c" && + command.at(-1)?.startsWith("/run/nemoclaw/runtime-state-mutation/") + ) { + return inspectTransportSession(command.at(-1) as string); + } if (command[2] === "--detach") { const transactionId = command.at(-1) ?? ""; brokerActive = true; @@ -647,6 +676,15 @@ function createContainerStateMutationHarness( if (conflict) throw new Error(conflict.stderr); return marker; }; + const releaseProviderWithoutTransportCleanup = () => { + if (!brokerActive || brokerTransactionId === null || marker === null) { + throw new Error("No active provider release transport exists."); + } + releasedMarker = marker; + marker = null; + brokerReleased = true; + state.supervisorStopped = false; + }; return { acquireRequests, authority, @@ -656,10 +694,16 @@ function createContainerStateMutationHarness( helperActions, supervisorSignals, transportBrokerActive: () => brokerActive, + transportBrokerSessionExists: (transactionId = brokerTransactionId) => + transactionId !== null && + [...transportFiles.keys()].some((file) => + file.startsWith(`/run/nemoclaw/runtime-state-mutation/${transactionId}/`), + ), transportCopySourceModes, lifecycleStore, lifecycleGeneration, owner, + releaseProviderWithoutTransportCleanup, replayDeferredAcquire, root, state, diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index 0735bd5dc88..e12533e4e86 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -9,7 +9,6 @@ import { describe, expect, it } from "vitest"; const ROOT = path.join(import.meta.dirname, "../.."); const PUBLISHER = path.join(ROOT, "scripts", "runtime_state_mutation_hermes_publisher.py"); -const STATE_PLAN = path.join(ROOT, "agents", "hermes", "state-lock-plan.json"); const START = path.join(ROOT, "agents", "hermes", "start.sh"); const HARNESS = String.raw` @@ -17,7 +16,6 @@ import hashlib import importlib.util import json import os -import shutil import sys import tempfile @@ -28,21 +26,26 @@ spec.loader.exec_module(publisher) publisher.ROOT_UID = os.getuid() publisher.ROOT_GID = os.getgid() -with open(sys.argv[2], "r", encoding="utf-8") as stream: - installed_value = json.load(stream) -installed_plan = {key: value for key, value in installed_value.items() if key != "$comment"} +installed_plan = { + "version": 1, + "readOnlyRoots": ["plugins", "workspace"], + "confidentialRoots": ["pairing"], + "readOnlyPrefixes": ["profile-"], + "confidentialPrefixes": ["secret-"], + "writableSubpaths": ["workspace/cache"], +} def canonical(value): return json.dumps(value, ensure_ascii=False, separators=(",", ":")) +def write_installed_plan(path): + with open(path, "w", encoding="utf-8") as stream: + stream.write(canonical(installed_plan)) + os.chmod(path, 0o444) + def marker(nonce="d" * 64, selectors=None, provider_id="docker"): - expected = [ - *["path:" + value for value in (".config-hash", ".env", "config.yaml")], - *["path:" + value for value in installed_plan["readOnlyRoots"]], - *["path:" + value for value in installed_plan["confidentialRoots"]], - *["prefix:" + value for value in installed_plan["readOnlyPrefixes"]], - *["prefix:" + value for value in installed_plan["confidentialPrefixes"]], - ] + expected = ["path:.config-hash", "path:.env", "path:config.yaml", "path:pairing", + "path:plugins", "path:workspace", "prefix:profile-", "prefix:secret-"] expected.sort(key=lambda value: value.encode()) selected = selectors or [ ({"kind": "path", "path": value.removeprefix("path:")} @@ -89,8 +92,7 @@ with tempfile.TemporaryDirectory() as temporary: durable = os.path.join(temporary, "durable") os.mkdir(durable, 0o711) plan_path = os.path.join(temporary, "state-lock-plan.json") - shutil.copyfile(sys.argv[2], plan_path) - os.chmod(plan_path, 0o444) + write_installed_plan(plan_path) publisher.DURABLE_DIRECTORY = durable publisher.STATE_LOCK_PLAN_PATH = plan_path publisher._verify_final_posture = lambda posture, plan_json: ( @@ -166,8 +168,7 @@ with tempfile.TemporaryDirectory() as temporary: durable = os.path.join(temporary, "durable") os.mkdir(durable, 0o711) plan_path = os.path.join(temporary, "state-lock-plan.json") - shutil.copyfile(sys.argv[2], plan_path) - os.chmod(plan_path, 0o444) + write_installed_plan(plan_path) publisher.DURABLE_DIRECTORY = durable publisher.STATE_LOCK_PLAN_PATH = plan_path publisher._verify_final_posture = lambda posture, plan_json: "4" * 64 @@ -204,8 +205,7 @@ with tempfile.TemporaryDirectory() as temporary: durable = os.path.join(temporary, "durable") os.mkdir(durable, 0o711) plan_path = os.path.join(temporary, "state-lock-plan.json") - shutil.copyfile(sys.argv[2], plan_path) - os.chmod(plan_path, 0o444) + write_installed_plan(plan_path) publisher.DURABLE_DIRECTORY = durable publisher.STATE_LOCK_PLAN_PATH = plan_path events = [] @@ -260,8 +260,7 @@ with tempfile.TemporaryDirectory() as temporary: durable = os.path.join(temporary, "durable") os.mkdir(durable, 0o711) plan_path = os.path.join(temporary, "state-lock-plan.json") - shutil.copyfile(sys.argv[2], plan_path) - os.chmod(plan_path, 0o444) + write_installed_plan(plan_path) publisher.DURABLE_DIRECTORY = durable publisher.STATE_LOCK_PLAN_PATH = plan_path events = [] @@ -319,7 +318,7 @@ print(json.dumps(results, sort_keys=True)) `; function runHarness(): Record { - const result = spawnSync("python3", ["-I", "-c", HARNESS, PUBLISHER, STATE_PLAN], { + const result = spawnSync("python3", ["-I", "-c", HARNESS, PUBLISHER], { encoding: "utf8", timeout: 20_000, }); @@ -475,12 +474,20 @@ describe("Hermes runtime state mutation publisher", () => { }); expect(result.retry).toMatchObject({ posture: "locked" }); expect(result.rollback).toMatchObject({ posture: "mutable" }); - const expectedStatePlan = JSON.parse(fs.readFileSync(STATE_PLAN, "utf8")) as Record< - string, - unknown - >; - delete expectedStatePlan.$comment; - expect(result.retry_events).toEqual([["verify", "locked", expectedStatePlan]]); + expect(result.retry_events).toEqual([ + [ + "verify", + "locked", + { + version: 1, + readOnlyRoots: ["plugins", "workspace"], + confidentialRoots: ["pairing"], + readOnlyPrefixes: ["profile-"], + confidentialPrefixes: ["secret-"], + writableSubpaths: ["workspace/cache"], + }, + ], + ]); expect(result.extra_selector).toBe("publisher-plan-selector-mismatch"); const events = result.events as Array<[string, string[]?]>; expect(events.filter(([action]) => action === "begin-shields-transition")).toHaveLength(2); diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index e3e5070569f..9165753d6a5 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -41,6 +41,7 @@ start = control.ProcessReference( fence = control.FenceProof(start, start, (), (os.geteuid(),)) marker = {"transactionId": "c" * 64, "nonce": "b" * 64} release_payload = b'{"release":"exact"}\n' +pending_release_ack_name = ".release-ack.json.pending" def code(operation): try: @@ -109,7 +110,7 @@ with tempfile.TemporaryDirectory() as root: payload = control._canonical_protocol_payload(expected) write_at( directory_fd, - control.STARTUP_RELEASE_ACK_PENDING_NAME, + pending_release_ack_name, payload, ) results["pendingIgnored"] = ( @@ -117,7 +118,7 @@ with tempfile.TemporaryDirectory() as root: is None ) os.rename( - control.STARTUP_RELEASE_ACK_PENDING_NAME, + pending_release_ack_name, control.STARTUP_RELEASE_ACK_NAME, src_dir_fd=directory_fd, dst_dir_fd=directory_fd, @@ -201,7 +202,7 @@ with tempfile.TemporaryDirectory() as root: ) write_at( directory_fd, - control.STARTUP_RELEASE_ACK_PENDING_NAME, + pending_release_ack_name, payload, ) finally: From 3667b6de40d266e62c1416d841c91fc5809a0e5b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 12:42:16 -0700 Subject: [PATCH 33/44] fix(runtime): bind pidfd to immutable task identity Signed-off-by: Prekshi Vyas --- scripts/runtime-state-mutation-control.py | 15 +++++- .../runtime-state-mutation-control-harness.ts | 52 +++++++++++++++++++ .../runtime-state-mutation-control.test.ts | 15 ++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index e85eb9c8fda..72e1e7cbcb9 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -212,6 +212,14 @@ def identity_key(self) -> tuple[object, ...]: self.proc_inode, ) + def kernel_task_key(self) -> tuple[object, ...]: + return ( + self.pid, + self.start_identity, + self.proc_device, + self.proc_inode, + ) + @dataclass(frozen=True) class ProcessReference: @@ -2743,7 +2751,12 @@ def _signal_exact_process(process: ProcessIdentity, requested_signal: int) -> No current = _capture_process(process.pid) if current is None: return - if current.identity_key() != process.identity_key(): + # The full process reference was authenticated before opening the + # pidfd. Recheck only immutable kernel task identity here: parent, + # credentials, and argv can change on the same task between the two + # observations. The pidfd remains bound to that task, while a real PID + # replacement changes its start time or procfs inode and fails closed. + if current.kernel_task_key() != process.kernel_task_key(): _fail("writer-pid-reused") try: signal.pidfd_send_signal(pidfd, requested_signal) diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index ea711f06eb7..fbd867e95a7 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -264,6 +264,58 @@ with tempfile.TemporaryDirectory() as process_probe: finally: control.os.stat = real_os_stat control._read_proc_file = real_read_proc_file +same_task_after_exec = process( + start.pid, + start.state, + start.parent_pid + 1, + start.start_identity, + 1000, + (b"/usr/local/bin/hermes", b"gateway"), + start.proc_inode, +) +replacement_task = process( + start.pid, + start.state, + start.parent_pid, + str(int(start.start_identity) + 1), + 1001, + start.command, + start.proc_inode + 1, +) +real_pidfd_open = getattr(control.os, "pidfd_open", None) +real_pidfd_send_signal = getattr(control.signal, "pidfd_send_signal", None) +real_os_close = control.os.close +pidfd_signal_events = [] +control.os.pidfd_open = lambda pid, flags: ( + pidfd_signal_events.append(["open", pid, flags]) or 91 +) +control.signal.pidfd_send_signal = lambda pidfd, requested: pidfd_signal_events.append( + ["signal", pidfd, requested] +) +control.os.close = lambda pidfd: pidfd_signal_events.append(["close", pidfd]) +try: + control._capture_process = lambda _pid: same_task_after_exec + results["same_task_signal"] = code( + lambda: real_signal_exact_process(start, signal.SIGSTOP) + ) + results["same_task_signal_events"] = list(pidfd_signal_events) + pidfd_signal_events.clear() + control._capture_process = lambda _pid: replacement_task + results["replacement_task_signal"] = code( + lambda: real_signal_exact_process(start, signal.SIGSTOP) + ) + results["replacement_task_signal_events"] = list(pidfd_signal_events) +finally: + control._capture_process = real_capture_process + if real_pidfd_open is None: + del control.os.pidfd_open + else: + control.os.pidfd_open = real_pidfd_open + if real_pidfd_send_signal is None: + del control.signal.pidfd_send_signal + else: + control.signal.pidfd_send_signal = real_pidfd_send_signal + control.os.close = real_os_close reference_signal_attempts = [] control._recapture_reference = lambda _reference, _code="fenced-process-drift": start def signal_reference_after_rescan(selected, requested): diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 34698859e5d..8298f080106 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -116,6 +116,21 @@ describe("runtime state mutation controller", () => { expect(harnessResult.reference_signal_timeout).toBe("writer-pid-reused"); }); + it("signals the same kernel task after mutable process metadata changes (#9485)", () => { + const sigstop = harnessResult.sigstop as number; + expect(harnessResult.same_task_signal).toBe("ok"); + expect(harnessResult.same_task_signal_events).toEqual([ + ["open", 10, 0], + ["signal", 91, sigstop], + ["close", 91], + ]); + expect(harnessResult.replacement_task_signal).toBe("writer-pid-reused"); + expect(harnessResult.replacement_task_signal_events).toEqual([ + ["open", 10, 0], + ["close", 91], + ]); + }); + it("publishes, rolls an activated fence back, and recovers every durable phase (#7744)", () => { expect(harnessResult).toMatchObject({ publish: "published", From 425301ff8de03d7b345b3496bdd1fd86abc8ffc5 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 13:53:58 -0700 Subject: [PATCH 34/44] fix(runtime): retain the pinned OpenShell supervisor Signed-off-by: Prekshi Vyas --- scripts/runtime-state-mutation-control.py | 61 +++++++++++++++---- .../runtime-state-mutation-control-harness.ts | 50 ++++++++++++++- .../runtime-state-mutation-control.test.ts | 10 ++- 3 files changed, 106 insertions(+), 15 deletions(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 72e1e7cbcb9..7056d345a43 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -2661,11 +2661,19 @@ def _process_has_reference_identity( process: ProcessIdentity, reference: ProcessReference ) -> bool: return bool( - process.pid == reference.pid - and process.start_identity == reference.start_identity + _process_has_kernel_task_reference(process, reference) and process.parent_pid == reference.parent_pid and process.uids == reference.uids and _process_command_sha256(process.command) == reference.command_sha256 + ) + + +def _process_has_kernel_task_reference( + process: ProcessIdentity, reference: ProcessReference +) -> bool: + return bool( + process.pid == reference.pid + and process.start_identity == reference.start_identity and process.proc_device == reference.proc_device and process.proc_inode == reference.proc_inode ) @@ -2680,6 +2688,22 @@ def _recapture_reference( return process +def _recapture_supervisor(reference: ProcessReference) -> ProcessIdentity: + process = _capture_process(reference.pid) + # PID 1 can refresh its displayed argv suffix while it remains the same + # kernel task. Revalidate the fixed OpenShell supervisor semantics instead + # of treating that mutable display metadata as task replacement. Start + # time and procfs identity still bind the original task and fail closed on + # a real replacement. + if ( + process is None + or not _process_has_kernel_task_reference(process, reference) + or not _is_openshell_supervisor(process) + ): + _fail("supervisor-identity-drift") + return process + + def _discover_fence(expected_mount_namespace: str) -> FenceProof: _assert_private_procfs() try: @@ -2704,9 +2728,7 @@ def _discover_fence(expected_mount_namespace: str) -> FenceProof: supervisor_reference = _process_reference(supervisor) start_reference = _process_reference(start) support_references = tuple(_process_reference(process) for process in support) - second_supervisor = _recapture_reference( - supervisor_reference, "supervisor-identity-drift" - ) + second_supervisor = _recapture_supervisor(supervisor_reference) second_writers = _capture_writer_processes(writer_uids) second_starts = [ process @@ -2813,7 +2835,7 @@ def _stop_reference(reference: ProcessReference) -> None: def _wait_for_host_stopped_supervisor(reference: ProcessReference) -> ProcessIdentity: deadline = time.monotonic() + PROCESS_STATE_SECONDS while True: - process = _recapture_reference(reference, "supervisor-identity-drift") + process = _recapture_supervisor(reference) if process.state in ("T", "t"): return process remaining = deadline - time.monotonic() @@ -2919,7 +2941,7 @@ def _prove_fence_shape( or fence.writer_uids != _supported_writer_uids() ): _fail("supervisor-identity-drift") - supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") + supervisor = _recapture_supervisor(fence.supervisor) start = _recapture_reference(fence.start, "start-process-identity-drift") support = tuple( _recapture_reference(reference, "startup-support-identity-drift") @@ -3882,7 +3904,7 @@ def _wait_for_activation( ) -> ActivationProof: deadline = time.monotonic() + ACTIVATION_SECONDS while True: - _recapture_reference(fence.supervisor, "supervisor-identity-drift") + _recapture_supervisor(fence.supervisor) gateway, _tree = _activation_tree(fence) if gateway is not None: try: @@ -3943,7 +3965,7 @@ def _freeze_activation( def _wait_for_startup_checkpoint(marker: dict[str, object], fence: FenceProof) -> str: deadline = time.monotonic() + ACTIVATION_SECONDS while True: - supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") + supervisor = _recapture_supervisor(fence.supervisor) start = _recapture_reference(fence.start, "start-process-identity-drift") selected = _read_startup_candidate(marker, fence, required=False) if selected is not None and start.state in ("T", "t"): @@ -4260,6 +4282,21 @@ def _resume_reference(reference: ProcessReference) -> ProcessIdentity: return _wait_for_reference_running(reference) +def _resume_supervisor(reference: ProcessReference) -> ProcessIdentity: + process = _recapture_supervisor(reference) + if process.state in ("T", "t"): + _signal_exact_process(process, signal.SIGCONT) + deadline = time.monotonic() + PROCESS_STATE_SECONDS + while True: + process = _recapture_supervisor(reference) + if process.state not in ("T", "t"): + return process + remaining = deadline - time.monotonic() + if remaining <= 0: + _fail("process-state-timeout") + time.sleep(min(POLL_SECONDS, remaining)) + + def _prove_released_activation( marker: dict[str, object], fence: FenceProof, activation: ActivationProof ) -> None: @@ -4273,7 +4310,7 @@ def _prove_released_activation( start = _recapture_reference(fence.start, "start-process-identity-drift") if start.state in ("T", "t"): _fail("start-process-stopped") - supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") + supervisor = _recapture_supervisor(fence.supervisor) if supervisor.state in ("T", "t"): _fail("supervisor-process-stopped") service_reference = next( @@ -4298,7 +4335,7 @@ def _prove_parent_acknowledged_activation( start = _recapture_reference(fence.start, "start-process-identity-drift") if start.state not in ("T", "t"): _fail("activation-release-parent-running") - supervisor = _recapture_reference(fence.supervisor, "supervisor-identity-drift") + supervisor = _recapture_supervisor(fence.supervisor) if supervisor.state in ("T", "t"): _fail("supervisor-process-stopped") service_reference = next( @@ -4338,7 +4375,7 @@ def _release_activation_hold(durable_fd: int, marker: dict[str, object]) -> None # Resume the exact pinned OpenShell supervisor last. Until the proven # workload is live, keeping PID 1 stopped prevents it from advertising a # transient Ready state or admitting unrelated sandbox commands. - _resume_reference(fence.supervisor) + _resume_supervisor(fence.supervisor) _wait_for_startup_release_ack(marker, fence, release_payload) _prove_parent_acknowledged_activation(marker, fence, activation) diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index fbd867e95a7..e3931d84e72 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -693,6 +693,44 @@ results["supervisor_identity_drift"] = fence_drift(1) results["start_identity_drift"] = fence_drift(10) results["startup_support_identity_drift"] = fence_drift(75) +def supervisor_refresh(selected): + control._capture_process = lambda pid: { + 1: selected, + 10: start, + 75: stdout_drain, + 76: stderr_drain, + }.get(pid) + control.os.readlink = lambda path: ( + "mnt:[401]" + if path == control.MOUNT_NAMESPACE_PATH + else real_readlink(path) + ) + try: + return code(lambda: real_prove_fence_shape(fence, "mnt:[401]")) + finally: + control.os.readlink = real_readlink + +refreshed_pid1 = process( + 1, + "T", + 0, + pid1.start_identity, + root_uid, + (control.OPENSHELL_ARGV0, b"--refreshed-status"), + pid1.proc_inode, +) +replaced_pid1 = process( + 1, + "T", + 0, + "101", + root_uid, + (control.OPENSHELL_ARGV0,), + 102, +) +results["refreshed_supervisor"] = supervisor_refresh(refreshed_pid1) +results["replaced_supervisor"] = supervisor_refresh(replaced_pid1) + hold_events = [] control._prove_fence_shape = lambda _fence, _mount: (pid1, start) control._stop_reference = lambda reference: hold_events.append(["stop", reference.pid]) @@ -714,6 +752,13 @@ control.PROCESS_STATE_SECONDS = 0 results["running_supervisor_hold"] = code(lambda: real_hold_exact_processes(fence, "mnt:[401]", activation)) control.PROCESS_STATE_SECONDS = 5 control._prove_fence_shape = lambda _fence, _mount: (stopped_pid1, start) +control._capture_process = lambda pid: stopped_pid1 if pid == 1 else { + 10: start, + 75: stdout_drain, + 76: stderr_drain, + 77: gateway, + 78: auxiliary, +}.get(pid) control._recapture_reference = lambda reference, _code="fenced-process-drift": stopped_pid1 if reference.pid == 1 else { 10: start, 75: stdout_drain, @@ -1138,7 +1183,7 @@ def state_process(original): original.proc_inode, ) by_release_pid = { - 1: pid1, + 1: refreshed_pid1, 10: start, 75: stdout_drain, 76: stderr_drain, @@ -1152,6 +1197,9 @@ control._prove_fence_shape = lambda _fence, _mount: ( control._recapture_reference = lambda reference, _code="fenced-process-drift": state_process( by_release_pid[reference.pid] ) +control._recapture_supervisor = lambda reference: state_process( + by_release_pid[reference.pid] +) def resume_reference(reference): if release_states[reference.pid] in ("T", "t"): release_events.append(["resume", reference.pid]) diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 8298f080106..99c5bbcd811 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -131,6 +131,11 @@ describe("runtime state mutation controller", () => { ]); }); + it("retains the same OpenShell supervisor across mutable argv metadata (#9485)", () => { + expect(harnessResult.refreshed_supervisor).toBe("ok"); + expect(harnessResult.replaced_supervisor).toBe("supervisor-identity-drift"); + }); + it("publishes, rolls an activated fence back, and recovers every durable phase (#7744)", () => { expect(harnessResult).toMatchObject({ publish: "published", @@ -242,6 +247,7 @@ describe("runtime state mutation controller", () => { }); it("records release intent before resuming exact writers and waits for parent acknowledgement (#10155)", () => { + const sigcont = harnessResult.sigcont as number; expect(harnessResult).toMatchObject({ release: "activation-proven", released_marker: true, @@ -259,7 +265,7 @@ describe("runtime state mutation controller", () => { ["resume", 77], ["resume", 78], ["resume", 10], - ["resume", 1], + ["signal", 1, sigcont], ["release-ack"], ["parent-ack-health"], ]); @@ -276,7 +282,7 @@ describe("runtime state mutation controller", () => { ["resume", 76], ["resume", 77], ["resume", 10], - ["resume", 1], + ["signal", 1, sigcont], ["release-ack"], ["parent-ack-health"], ]); From a4f1e663528e6d8b78a7cab4c1a703e14fa5cb16 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 14:09:12 -0700 Subject: [PATCH 35/44] fix(shields): preserve the production settle window Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 2 +- src/lib/shields/relock-reconfirm.test.ts | 16 +++++++++++++--- src/lib/shields/relock-reconfirm.ts | 15 +++++++++------ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 69510c12fa6..872303e3ea7 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -5560,7 +5560,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | -| `NEMOCLAW_SHIELDS_SETTLE_MS` | milliseconds (default `750`, clamped to `0` to `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | +| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive milliseconds (default `750`, maximum `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Zero, negative, blank, and invalid values use the default. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | | `NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS` | `1` to opt in | Makes `$$nemoclaw uninstall` remove every gateway port on the host instead of only the port `NEMOCLAW_GATEWAY_PORT` selects. Equivalent to passing the `--all-gateway-ports` flag; the whole-host `Proceed?` confirmation still applies unless `--yes` is also passed. Each port runs as its own uninstall, and the variable is dropped from those runs so the sweep cannot re-enter itself. | | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA` | `1` to opt in | Acknowledges data loss during `$$nemoclaw uninstall`, skips eligible fresh sandbox backups, and removes the otherwise-preserved entries (`rebuild-backups/`, `backups/`, `sandboxes.json`) in the selected gateway's state root. It does not select the explicit `--destroy-user-data` CLI-shim removal path; shim handling follows the ordinary selected-gateway scope. The global `Proceed?` confirmation still applies unless `--yes` is also passed. | diff --git a/src/lib/shields/relock-reconfirm.test.ts b/src/lib/shields/relock-reconfirm.test.ts index 030cf73de22..906144abfbb 100644 --- a/src/lib/shields/relock-reconfirm.test.ts +++ b/src/lib/shields/relock-reconfirm.test.ts @@ -155,11 +155,21 @@ describe("resolveSettleMs", () => { expect(resolveSettleMs()).toBe(10_000); }); - it("clamps a negative env value to 0", () => { + it.each(["0", "-500", "0.5", " ", "invalid"])( + "keeps the production settle window for a non-positive or invalid env value (%s)", + (value) => { + delete process.env.VITEST; + process.env.NODE_ENV = "production"; + process.env.NEMOCLAW_SHIELDS_SETTLE_MS = value; + expect(resolveSettleMs()).toBe(750); + }, + ); + + it("honours the minimum positive integer env value", () => { delete process.env.VITEST; process.env.NODE_ENV = "production"; - process.env.NEMOCLAW_SHIELDS_SETTLE_MS = "-500"; - expect(resolveSettleMs()).toBe(0); + process.env.NEMOCLAW_SHIELDS_SETTLE_MS = "1"; + expect(resolveSettleMs()).toBe(1); }); it("honours a valid in-range env value", () => { diff --git a/src/lib/shields/relock-reconfirm.ts b/src/lib/shields/relock-reconfirm.ts index 3a7a018b727..70d345232f6 100644 --- a/src/lib/shields/relock-reconfirm.ts +++ b/src/lib/shields/relock-reconfirm.ts @@ -33,7 +33,6 @@ export { waitForHermesInferenceRouteConvergence } from "./inference-convergence" const DEFAULT_MAX_ATTEMPTS = 3; const DEFAULT_SETTLE_MS = 750; -const MIN_SETTLE_MS = 0; const MAX_SETTLE_MS = 10_000; /** Result of a single `lockAgentConfig` call: apply + verify. */ @@ -70,9 +69,9 @@ export interface RelockReconfirmResult { /** * Resolve the settle window (ms) between applying a lock and re-confirming it. * - * Reads `NEMOCLAW_SHIELDS_SETTLE_MS`, defaulting to 750ms and clamping to - * [0, 10000]. Returns 0 only under Vitest so suites don't incur real blocking - * waits. + * Reads `NEMOCLAW_SHIELDS_SETTLE_MS`, defaulting to 750ms. Positive values are + * capped at 10000ms; invalid or non-positive values use the default. Returns 0 + * only under Vitest so suites don't incur real blocking waits. */ export function resolveSettleMs(): number { // VITEST is the precise test signal (Vitest always sets it). Do NOT key off @@ -82,14 +81,18 @@ export function resolveSettleMs(): number { return 0; } const raw = process.env.NEMOCLAW_SHIELDS_SETTLE_MS; - if (raw === undefined || raw === "") { + if (raw === undefined || raw.trim() === "") { return DEFAULT_SETTLE_MS; } const parsed = Number(raw); if (!Number.isFinite(parsed)) { return DEFAULT_SETTLE_MS; } - return Math.min(MAX_SETTLE_MS, Math.max(MIN_SETTLE_MS, Math.trunc(parsed))); + const settleMs = Math.trunc(parsed); + if (settleMs <= 0) { + return DEFAULT_SETTLE_MS; + } + return Math.min(MAX_SETTLE_MS, settleMs); } /** From 662868102dfe90b452cf87e583700bc80380a5ee Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 14:21:56 -0700 Subject: [PATCH 36/44] test(shields): reuse the Hermes publisher harness Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 2 +- ...runtime-state-mutation-hermes-publisher.test.ts | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 872303e3ea7..8784a501f60 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -5560,7 +5560,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | -| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive milliseconds (default `750`, maximum `10000`) | Settle window NemoClaw waits after re-applying a config lockdown (during shields auto-restore and `$$nemoclaw shields up` drift remediation) before re-confirming the lock still holds. Zero, negative, blank, and invalid values use the default. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | +| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive whole-number milliseconds (default `750`, maximum `10000`) | Settle window NemoClaw waits after re-applying a config lockdown during ordinary `$$nemoclaw shields up` transitions, shields auto-restore, and `shields up` drift remediation before re-confirming that the lock still holds. Fractional values below `1`, zero, negative, blank, and invalid values use the default. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | | `NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS` | `1` to opt in | Makes `$$nemoclaw uninstall` remove every gateway port on the host instead of only the port `NEMOCLAW_GATEWAY_PORT` selects. Equivalent to passing the `--all-gateway-ports` flag; the whole-host `Proceed?` confirmation still applies unless `--yes` is also passed. Each port runs as its own uninstall, and the variable is dropped from those runs so the sweep cannot re-enter itself. | | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA` | `1` to opt in | Acknowledges data loss during `$$nemoclaw uninstall`, skips eligible fresh sandbox backups, and removes the otherwise-preserved entries (`rebuild-backups/`, `backups/`, `sandboxes.json`) in the selected gateway's state root. It does not select the explicit `--destroy-user-data` CLI-shim removal path; shim handling follows the ordinary selected-gateway scope. The global `Proceed?` confirmation still applies unless `--yes` is also passed. | diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index e12533e4e86..666404e1976 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -5,7 +5,7 @@ import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { beforeAll, describe, expect, it } from "vitest"; const ROOT = path.join(import.meta.dirname, "../.."); const PUBLISHER = path.join(ROOT, "scripts", "runtime_state_mutation_hermes_publisher.py"); @@ -460,8 +460,14 @@ function entrypointGateEnvironment( } describe("Hermes runtime state mutation publisher", () => { + let harnessResult: Record; + + beforeAll(() => { + harnessResult = runHarness(); + }); + it("publishes and rolls back only the exact installed full plan (#7744)", () => { - const result = runHarness(); + const result = harnessResult; expect(result.first).toMatchObject({ protocol: "nemoclaw-runtime-state-mutation-publisher-v1", posture: "locked", @@ -506,7 +512,7 @@ describe("Hermes runtime state mutation publisher", () => { }); it("recovers nonce-bound begin and finish response loss without replaying them (#7744)", () => { - const result = runHarness(); + const result = harnessResult; expect(result).toMatchObject({ begin_loss: "simulated-begin-response-loss", finish_loss: "simulated-finish-response-loss", @@ -519,7 +525,7 @@ describe("Hermes runtime state mutation publisher", () => { }); it("can publish the exact rollback after a target begin response is lost (#7744)", () => { - const result = runHarness(); + const result = harnessResult; expect(result).toMatchObject({ rollback_begin_loss: "simulated-begin-response-loss", rollback_after_begin_loss: { posture: "mutable", nonce: "5".repeat(64) }, From 3361440c36abc18e10dc9ff30f1b4bee8782c485 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 14:48:54 -0700 Subject: [PATCH 37/44] test(shields): close advisor feedback Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 2 +- .../docker-state-mutation.test.ts | 336 +++++++++--------- .../runtime-provider/docker-state-mutation.ts | 36 +- src/lib/shields/relock-reconfirm.test.ts | 4 +- src/lib/shields/relock-reconfirm.ts | 15 +- 5 files changed, 198 insertions(+), 195 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 8784a501f60..994dacada4d 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -5560,7 +5560,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | -| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive whole-number milliseconds (default `750`, maximum `10000`) | Settle window NemoClaw waits after re-applying a config lockdown during ordinary `$$nemoclaw shields up` transitions, shields auto-restore, and `shields up` drift remediation before re-confirming that the lock still holds. Fractional values below `1`, zero, negative, blank, and invalid values use the default. Detects when an in-sandbox reconciler changes config file permissions after lockdown and re-applies the lock; if NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This narrows the window in which a reconciler can revert permissions rather than eliminating it. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise it on hosts where the gateway settles slowly. | +| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive whole-number milliseconds (default `750`, maximum `10000`) | NemoClaw waits this long after re-applying a config lockdown before checking that the lock still holds. It applies during ordinary `$$nemoclaw shields up` transitions, shields auto-restore, and `shields up` drift remediation. Fractional, zero, negative, blank, and invalid values use the default. If NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This check narrows the window in which an in-sandbox reconciler can revert permissions; it does not eliminate that window. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise the value on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | | `NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS` | `1` to opt in | Makes `$$nemoclaw uninstall` remove every gateway port on the host instead of only the port `NEMOCLAW_GATEWAY_PORT` selects. Equivalent to passing the `--all-gateway-ports` flag; the whole-host `Proceed?` confirmation still applies unless `--yes` is also passed. Each port runs as its own uninstall, and the variable is dropped from those runs so the sweep cannot re-enter itself. | | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA` | `1` to opt in | Acknowledges data loss during `$$nemoclaw uninstall`, skips eligible fresh sandbox backups, and removes the otherwise-preserved entries (`rebuild-backups/`, `backups/`, `sandboxes.json`) in the selected gateway's state root. It does not select the explicit `--destroy-user-data` CLI-shim removal path; shim handling follows the ordinary selected-gateway scope. The global `Proceed?` confirmation still applies unless `--yes` is also passed. | diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index a95ccc1ba49..9e2c0c75cda 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { execFileSync, spawn } from "node:child_process"; +import { spawn } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; @@ -26,7 +26,7 @@ import { import { createDockerOperationAuthority } from "./docker-operation-authority"; import { DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS, - DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE, + createDockerStateMutationHelperTransportBrokerSource, createDockerStateMutationOwner, createDockerStateMutationSurface, } from "./docker-state-mutation"; @@ -77,197 +77,179 @@ async function waitForPath(filePath: string, timeoutMs: number): Promise { throw new Error(`timed out waiting for ${path.basename(filePath)}`); } +type BrokerAction = + | "acquire" + | "assert" + | "publish" + | "recover" + | "rollback" + | "activate" + | "release"; + +interface BrokerResponse { + readonly schemaVersion: number; + readonly action: BrokerAction; + readonly identity: string; + readonly status: number; + readonly stdout: string; + readonly stderr: string; +} + +async function createBrokerRuntime( + helperSource: (root: string) => string, + activateTimeoutSeconds = 0.25, +) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-state-mutation-broker-")); + const helper = path.join(root, "helper.py"); + const transaction = randomBytes(32).toString("hex"); + const session = path.join(root, transaction); + const uid = process.getuid?.() ?? 0; + const gid = process.getgid?.() ?? 0; + const brokerSource = createDockerStateMutationHelperTransportBrokerSource({ + root, + expectedUid: uid, + expectedGid: gid, + activateTimeoutSeconds, + }); + fs.writeFileSync(helper, helperSource(root), { mode: 0o500 }); + const broker = spawn("python3", ["-I", "-c", brokerSource, helper, transaction], { + stdio: ["ignore", "ignore", "pipe"], + }); + const brokerExit = new Promise((resolve, reject) => { + broker.once("exit", () => resolve()); + broker.once("error", reject); + }); + let brokerStderr = ""; + broker.stderr.setEncoding("utf8"); + broker.stderr.on("data", (chunk: string) => { + brokerStderr += chunk; + }); + + try { + await waitForPath(path.join(session, "ready"), 2_000); + } catch (error: unknown) { + broker.kill("SIGTERM"); + await brokerExit; + fs.rmSync(root, { force: true, recursive: true }); + throw error; + } + + return { + helper, + root, + session, + transaction, + stderr: () => brokerStderr, + close: async () => { + broker.kill("SIGTERM"); + await brokerExit; + fs.rmSync(root, { force: true, recursive: true }); + }, + }; +} + +async function sendBrokerRequest( + runtime: Awaited>, + action: BrokerAction, +): Promise<{ readonly elapsedMs: number; readonly response: BrokerResponse }> { + const request = Buffer.from( + `${JSON.stringify({ action, transactionId: runtime.transaction })}\n`, + "utf8", + ); + const identity = createHash("sha256").update(request).digest("hex"); + const responsePath = path.join(runtime.session, `${identity}.response`); + const startedAt = Date.now(); + fs.writeFileSync(path.join(runtime.session, `${identity}.${action}.incoming`), request, { + mode: 0o600, + }); + await waitForPath(responsePath, 1_500); + return { + elapsedMs: Date.now() - startedAt, + response: JSON.parse(fs.readFileSync(responsePath, "utf8")) as BrokerResponse, + }; +} + +function brokerFailureCode(response: BrokerResponse): string { + return (JSON.parse(response.stderr) as { code: string }).code; +} + afterEach(() => { cleanupDockerStateMutationRoots(); }); describe("Docker runtime-provider state mutation surface", () => { - it("preserves safe broker diagnostics after request validation", () => { - const definitionsEnd = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.indexOf( - "\nhelper = sys.argv[1]\n", - ); - expect(definitionsEnd).toBeGreaterThan(0); - const definitions = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.slice( - 0, - definitionsEnd, + it("publishes safe full-broker diagnostics after request validation", async () => { + const runtime = await createBrokerRuntime( + () => `import os,sys +action = sys.argv[1] +if action == "acquire": + os.write(2, b"raw helper failure") + raise SystemExit(2) +if action == "assert": + os.write(2, b"unexpected helper stderr") +if action == "publish": + os.write(1, b"\\xff") +`, ); - const probe = `${definitions} -helper = "/definitely-missing/nemoclaw-runtime-state-mutation-control.py" -try: - run_helper("acquire", b"{}\\n") -except (OSError, RuntimeError, UnicodeError, ValueError) as error: - missing_helper = post_validation_failure_code(error) -print(json.dumps({ - "missingHelper": missing_helper, - "permission": post_validation_failure_code(PermissionError()), - "encoding": post_validation_failure_code(UnicodeDecodeError("utf-8", b"x", 0, 1, "invalid")), - "invalidResponse": post_validation_failure_code(ValueError()), - "helperProcess": json.loads(normalize_helper_stderr("acquire", 2, b"raw python error"))["code"], - "helperProtocol": json.loads(normalize_helper_stderr("acquire", 0, b"unexpected stderr"))["code"], - "timeout": json.loads(failure_stderr("acquire", "helper-timeout"))["code"], - "activateTimeoutSeconds": TIMEOUTS["activate"], -}, separators=(",", ":"))) -`; - expect( - JSON.parse( - execFileSync("python3", ["-I", "-c", probe], { - encoding: "utf8", - timeout: 5_000, - }), - ), - ).toEqual({ - missingHelper: "helper-file-missing", - permission: "transport-permission-denied", - encoding: "transport-response-encoding-invalid", - invalidResponse: "transport-response-invalid", - helperProcess: "helper-process-failed", - helperProtocol: "helper-protocol-stderr", - timeout: "helper-timeout", - activateTimeoutSeconds: DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS / 1000, - }); - }); - - it("terminates the isolated helper process group before reporting a timeout", () => { - const definitionsEnd = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.indexOf( - "\nhelper = sys.argv[1]\n", - ); - const definitions = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.slice( - 0, - definitionsEnd, - ); - const probe = `${definitions} -import tempfile -root = tempfile.mkdtemp(prefix="nemoclaw-helper-timeout-") -helper = os.path.join(root, "helper.py") -with open(helper, "w", encoding="utf-8") as stream: - stream.write("import os,time\\n") - stream.write("if os.fork() == 0: time.sleep(30)\\n") - stream.write("time.sleep(30)\\n") -original_lstat = os.lstat -class RootHelperMetadata: - st_mode = stat.S_IFREG | 0o555 - st_uid = 0 - st_gid = 0 -os.lstat = lambda target: RootHelperMetadata() if target == helper else original_lstat(target) -TIMEOUTS["activate"] = 0.05 -try: - run_helper("activate", b"{}\\n") -except subprocess.TimeoutExpired: - print("helper-timeout") -`; + try { + const failed = await sendBrokerRequest(runtime, "acquire"); + expect(brokerFailureCode(failed.response)).toBe("helper-process-failed"); - expect( - execFileSync("python3", ["-I", "-c", probe], { - encoding: "utf8", - timeout: 5_000, - }).trim(), - ).toBe("helper-timeout"); - }); + const protocol = await sendBrokerRequest(runtime, "assert"); + expect(brokerFailureCode(protocol.response)).toBe("helper-protocol-stderr"); - it("keeps the signal replay inside one activation broker deadline (#10155)", () => { - const definitionsEnd = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.indexOf( - "\nhelper = sys.argv[1]\n", - ); - const definitions = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.slice( - 0, - definitionsEnd, - ); - const probe = `${definitions} -import types -helper = "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py" -os.lstat = lambda _path: types.SimpleNamespace( - st_mode=stat.S_IFREG | 0o444, st_uid=0, st_gid=0) -ticks = iter((100.0, 100.0, 325.0)) -time.monotonic = lambda: next(ticks) -timeouts = [] -class FakeProcess: - def __init__(self, args, **_kwargs): - self.args = args - self.pid = 1 - self.returncode = -9 if not timeouts else 0 - def communicate(self, _request, timeout): - timeouts.append(timeout) - return (b"", b"") -subprocess.Popen = FakeProcess -run_helper("activate", b"{}\\n") -print(json.dumps(timeouts)) -`; + const encoding = await sendBrokerRequest(runtime, "publish"); + expect(brokerFailureCode(encoding.response)).toBe("transport-response-encoding-invalid"); - expect( - JSON.parse( - execFileSync("python3", ["-I", "-c", probe], { - encoding: "utf8", - timeout: 5_000, - }), - ), - ).toEqual([485, 260]); + fs.unlinkSync(runtime.helper); + const missing = await sendBrokerRequest(runtime, "recover"); + expect(brokerFailureCode(missing.response)).toBe("helper-file-missing"); + expect(runtime.stderr(), runtime.stderr()).toBe(""); + } finally { + await runtime.close(); + } }); - it("publishes a bounded activation timeout after the helper deadline (#10155)", async () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-state-mutation-broker-")); - const helper = path.join(root, "slow-helper.py"); - const transaction = randomBytes(32).toString("hex"); - const session = path.join(root, transaction); - const uid = process.getuid?.() ?? 0; - const gid = process.getgid?.() ?? 0; - const brokerSource = DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE.replace( - 'ROOT = "/run/nemoclaw/runtime-state-mutation"', - `ROOT = ${JSON.stringify(root)}`, - ) - .replace(/"activate": [0-9.]+/u, '"activate": 0.05') - .replaceAll( - "metadata.st_uid != 0 or metadata.st_gid != 0", - `metadata.st_uid != ${uid} or metadata.st_gid != ${gid}`, - ) - .replaceAll( - "before.st_uid != 0 or before.st_gid != 0", - `before.st_uid != ${uid} or before.st_gid != ${gid}`, - ); - fs.writeFileSync(helper, "import time\ntime.sleep(1)\n", { mode: 0o500 }); - const broker = spawn("python3", ["-I", "-c", brokerSource, helper, transaction], { - stdio: ["ignore", "ignore", "pipe"], - }); - const brokerExit = new Promise((resolve, reject) => { - broker.once("exit", () => resolve()); - broker.once("error", reject); - }); - let brokerStderr = ""; - broker.stderr.setEncoding("utf8"); - broker.stderr.on("data", (chunk: string) => { - brokerStderr += chunk; + it("bounds signal replay and terminates the full helper process group (#10155)", async () => { + let leakedChildMarker = ""; + const runtime = await createBrokerRuntime((root) => { + const countPath = path.join(root, "helper-count"); + leakedChildMarker = path.join(root, "leaked-child"); + return `import os,signal,sys,time +count_path = ${JSON.stringify(countPath)} +leaked_child_marker = ${JSON.stringify(leakedChildMarker)} +if sys.argv[1] != "activate": + raise SystemExit(0) +try: + with open(count_path, "r", encoding="utf-8") as stream: + count = int(stream.read()) +except FileNotFoundError: + count = 0 +with open(count_path, "w", encoding="utf-8") as stream: + stream.write(str(count + 1)) +if count == 0: + time.sleep(0.2) + os.kill(os.getpid(), signal.SIGKILL) +if os.fork() == 0: + time.sleep(0.35) + with open(leaked_child_marker, "w", encoding="utf-8") as stream: + stream.write("helper process group survived") + os._exit(0) +time.sleep(30) +`; }); try { - await waitForPath(path.join(session, "ready"), 2_000); - const request = Buffer.from( - `${JSON.stringify({ action: "activate", transactionId: transaction })}\n`, - "utf8", - ); - const identity = createHash("sha256").update(request).digest("hex"); - const responsePath = path.join(session, `${identity}.response`); - const startedAt = Date.now(); - fs.writeFileSync(path.join(session, `${identity}.activate.incoming`), request, { - mode: 0o600, - }); - await waitForPath(responsePath, 1_000); - - const response = JSON.parse(fs.readFileSync(responsePath, "utf8")); - expect(response).toEqual({ - schemaVersion: 1, - action: "activate", - identity, - status: 1, - stdout: "", - stderr: - '{"schemaVersion":1,"action":"activate","status":"failed","code":"helper-timeout"}\n', - }); - expect(Date.now() - startedAt).toBeLessThan(500); - expect(brokerStderr, brokerStderr).toBe(""); + const result = await sendBrokerRequest(runtime, "activate"); + expect(result.response).toMatchObject({ action: "activate", status: 1, stdout: "" }); + expect(brokerFailureCode(result.response)).toBe("helper-timeout"); + expect(result.elapsedMs).toBeLessThan(400); + await new Promise((resolve) => setTimeout(resolve, 450)); + expect(fs.existsSync(leakedChildMarker)).toBe(false); + expect(runtime.stderr(), runtime.stderr()).toBe(""); } finally { - broker.kill("SIGTERM"); - await brokerExit; - fs.rmSync(root, { force: true, recursive: true }); + await runtime.close(); } }); diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index 0db4d33a3ea..e413ece591a 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -103,7 +103,17 @@ const POSITIVE_DECIMAL = /^[1-9][0-9]*$/u; const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; const helperTransportPoll = new Int32Array(new SharedArrayBuffer(4)); -export const DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE = String.raw` +export interface DockerStateMutationHelperTransportBrokerSourceOptions { + readonly root: string; + readonly expectedUid: number; + readonly expectedGid: number; + readonly activateTimeoutSeconds: number; +} + +export function createDockerStateMutationHelperTransportBrokerSource( + options: DockerStateMutationHelperTransportBrokerSourceOptions, +): string { + return String.raw` import fcntl import hashlib import json @@ -115,9 +125,11 @@ import subprocess import sys import time -ROOT = "/run/nemoclaw/runtime-state-mutation" +ROOT = ${JSON.stringify(options.root)} +EXPECTED_UID = ${options.expectedUid} +EXPECTED_GID = ${options.expectedGid} MAXIMUM = 128 * 1024 -TIMEOUTS = {"acquire": 30, "assert": 30, "publish": 900, "recover": 900, "rollback": 900, "activate": ${DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS / 1000}, "release": ${HELPER_RELEASE_TIMEOUT_MS / 1000}} +TIMEOUTS = {"acquire": 30, "assert": 30, "publish": 900, "recover": 900, "rollback": 900, "activate": ${options.activateTimeoutSeconds}, "release": ${HELPER_RELEASE_TIMEOUT_MS / 1000}} IDENTITY = re.compile(r"[a-f0-9]{64}\Z") INCOMING = re.compile(r"([a-f0-9]{64})\.(acquire|assert|publish|recover|rollback|activate|release)\.incoming\Z") PUBLICATION_SETTLE_SECONDS = 5 @@ -127,7 +139,8 @@ def fail(code): def directory(path): metadata = os.lstat(path) - if (not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != 0 or metadata.st_gid != 0 or + if (not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != EXPECTED_UID or + metadata.st_gid != EXPECTED_GID or stat.S_IMODE(metadata.st_mode) != 0o700): fail("transport-directory-invalid") @@ -152,7 +165,8 @@ def private_file(path): before = os.fstat(descriptor) payload = os.read(descriptor, MAXIMUM + 1) after = os.fstat(descriptor) - if (not stat.S_ISREG(before.st_mode) or before.st_uid != 0 or before.st_gid != 0 or + if (not stat.S_ISREG(before.st_mode) or before.st_uid != EXPECTED_UID or + before.st_gid != EXPECTED_GID or stat.S_IMODE(before.st_mode) != 0o600 or before.st_nlink != 1 or len(payload) > MAXIMUM or os.read(descriptor, 1) or (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_uid, @@ -251,7 +265,8 @@ def run_helper(action, request): metadata = os.lstat(helper) except FileNotFoundError: fail("helper-file-missing") - if (not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != 0 or metadata.st_gid != 0 or + if (not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != EXPECTED_UID or + metadata.st_gid != EXPECTED_GID or stat.S_IMODE(metadata.st_mode) & 0o022): fail("helper-file-invalid") deadline = time.monotonic() + TIMEOUTS[action] @@ -392,6 +407,15 @@ while True: pass time.sleep(0.05) `; +} + +export const DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE = + createDockerStateMutationHelperTransportBrokerSource({ + root: HELPER_TRANSPORT_ROOT, + expectedUid: 0, + expectedGid: 0, + activateTimeoutSeconds: DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS / 1000, + }); type HelperAction = | "acquire" diff --git a/src/lib/shields/relock-reconfirm.test.ts b/src/lib/shields/relock-reconfirm.test.ts index 906144abfbb..5e8658e3b5d 100644 --- a/src/lib/shields/relock-reconfirm.test.ts +++ b/src/lib/shields/relock-reconfirm.test.ts @@ -155,8 +155,8 @@ describe("resolveSettleMs", () => { expect(resolveSettleMs()).toBe(10_000); }); - it.each(["0", "-500", "0.5", " ", "invalid"])( - "keeps the production settle window for a non-positive or invalid env value (%s)", + it.each(["0", "-500", "0.5", "1.5", "750.1", " ", "invalid"])( + "keeps the production settle window for a non-positive, fractional, blank, or invalid env value (%s)", (value) => { delete process.env.VITEST; process.env.NODE_ENV = "production"; diff --git a/src/lib/shields/relock-reconfirm.ts b/src/lib/shields/relock-reconfirm.ts index 70d345232f6..3df6b697e33 100644 --- a/src/lib/shields/relock-reconfirm.ts +++ b/src/lib/shields/relock-reconfirm.ts @@ -69,9 +69,10 @@ export interface RelockReconfirmResult { /** * Resolve the settle window (ms) between applying a lock and re-confirming it. * - * Reads `NEMOCLAW_SHIELDS_SETTLE_MS`, defaulting to 750ms. Positive values are - * capped at 10000ms; invalid or non-positive values use the default. Returns 0 - * only under Vitest so suites don't incur real blocking waits. + * Reads `NEMOCLAW_SHIELDS_SETTLE_MS`, defaulting to 750ms. Positive whole + * numbers are capped at 10000ms; invalid, fractional, or non-positive values + * use the default. Returns 0 only under Vitest so suites don't incur real + * blocking waits. */ export function resolveSettleMs(): number { // VITEST is the precise test signal (Vitest always sets it). Do NOT key off @@ -85,14 +86,10 @@ export function resolveSettleMs(): number { return DEFAULT_SETTLE_MS; } const parsed = Number(raw); - if (!Number.isFinite(parsed)) { + if (!Number.isInteger(parsed) || parsed <= 0) { return DEFAULT_SETTLE_MS; } - const settleMs = Math.trunc(parsed); - if (settleMs <= 0) { - return DEFAULT_SETTLE_MS; - } - return Math.min(MAX_SETTLE_MS, settleMs); + return Math.min(MAX_SETTLE_MS, parsed); } /** From 99c4c725a2bfcb1fdbfaa05b63995acd04276cc8 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 15:00:34 -0700 Subject: [PATCH 38/44] fix(runtime): bind supervisor executable identity Signed-off-by: Prekshi Vyas --- scripts/runtime-state-mutation-control.py | 57 +++++++++++++++++-- .../runtime-state-mutation-startup-gate.py | 14 +++++ .../runtime-state-mutation-control-harness.ts | 37 +++++++++++- .../runtime-state-mutation-control.test.ts | 15 ++++- ...runtime-state-mutation-release-ack.test.ts | 8 +++ ...untime-state-mutation-startup-gate.test.ts | 4 ++ 6 files changed, 126 insertions(+), 9 deletions(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 7056d345a43..d50df81db5d 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -200,6 +200,8 @@ class ProcessIdentity: command: tuple[bytes, ...] proc_device: int proc_inode: int + executable_device: int + executable_inode: int def identity_key(self) -> tuple[object, ...]: return ( @@ -210,6 +212,8 @@ def identity_key(self) -> tuple[object, ...]: self.command, self.proc_device, self.proc_inode, + self.executable_device, + self.executable_inode, ) def kernel_task_key(self) -> tuple[object, ...]: @@ -220,6 +224,9 @@ def kernel_task_key(self) -> tuple[object, ...]: self.proc_inode, ) + def executable_key(self) -> tuple[int, int]: + return (self.executable_device, self.executable_inode) + @dataclass(frozen=True) class ProcessReference: @@ -230,6 +237,8 @@ class ProcessReference: command_sha256: str proc_device: int proc_inode: int + executable_device: int + executable_inode: int @dataclass(frozen=True) @@ -886,6 +895,8 @@ def _parse_request(action: Action, raw: bytes) -> Request: "commandSha256", "procDevice", "procInode", + "executableDevice", + "executableInode", ) FENCE_KEYS = ("supervisor", "start", "startSupport", "writerUids") @@ -965,6 +976,8 @@ def _process_reference(process: ProcessIdentity) -> ProcessReference: _process_command_sha256(process.command), process.proc_device, process.proc_inode, + process.executable_device, + process.executable_inode, ) @@ -977,6 +990,8 @@ def _process_reference_payload(reference: ProcessReference) -> dict[str, object] "commandSha256": reference.command_sha256, "procDevice": str(reference.proc_device), "procInode": str(reference.proc_inode), + "executableDevice": str(reference.executable_device), + "executableInode": str(reference.executable_inode), } @@ -997,7 +1012,14 @@ def _process_reference_from_value(value: object, code: str) -> ProcessReference: command_sha256 = _hex_digest(reference["commandSha256"], code) proc_device = _decimal_identity(reference["procDevice"], code) proc_inode = _decimal_identity(reference["procInode"], code) - if proc_device == "0" or proc_inode == "0": + executable_device = _decimal_identity(reference["executableDevice"], code) + executable_inode = _decimal_identity(reference["executableInode"], code) + if ( + proc_device == "0" + or proc_inode == "0" + or executable_device == "0" + or executable_inode == "0" + ): _fail(code) return ProcessReference( pid, @@ -1007,6 +1029,8 @@ def _process_reference_from_value(value: object, code: str) -> ProcessReference: command_sha256, int(proc_device, 10), int(proc_inode, 10), + int(executable_device, 10), + int(executable_inode, 10), ) @@ -2509,12 +2533,15 @@ def _assert_private_procfs() -> None: def _capture_process(pid: int) -> ProcessIdentity | None: process_path = os.path.join(PROC_ROOT, str(pid)) + executable_path = os.path.join(process_path, "exe") try: before = os.stat(process_path, follow_symlinks=False) + executable_before = os.stat(executable_path) first_stat = _read_proc_file(os.path.join(process_path, "stat")) status = _read_proc_file(os.path.join(process_path, "status")) command_raw = _read_proc_file(os.path.join(process_path, "cmdline")) second_stat = _read_proc_file(os.path.join(process_path, "stat")) + executable_after = os.stat(executable_path) after = os.stat(process_path, follow_symlinks=False) except (FileNotFoundError, ProcessLookupError): return None @@ -2527,6 +2554,7 @@ def _capture_process(pid: int) -> ProcessIdentity | None: if ( not stat.S_ISDIR(before.st_mode) or _stable_stat(before) != _stable_stat(after) + or not _same_filesystem_object(executable_before, executable_after) or (parent, start) != (second_parent, second_start) ): _fail("raced-writer-process") @@ -2539,6 +2567,8 @@ def _capture_process(pid: int) -> ProcessIdentity | None: tuple(part for part in command_raw.split(b"\0") if part), before.st_dev, before.st_ino, + executable_after.st_dev, + executable_after.st_ino, ) @@ -2665,6 +2695,16 @@ def _process_has_reference_identity( and process.parent_pid == reference.parent_pid and process.uids == reference.uids and _process_command_sha256(process.command) == reference.command_sha256 + and _process_has_reference_executable(process, reference) + ) + + +def _process_has_reference_executable( + process: ProcessIdentity, reference: ProcessReference +) -> bool: + return bool( + process.executable_device == reference.executable_device + and process.executable_inode == reference.executable_inode ) @@ -2693,11 +2733,12 @@ def _recapture_supervisor(reference: ProcessReference) -> ProcessIdentity: # PID 1 can refresh its displayed argv suffix while it remains the same # kernel task. Revalidate the fixed OpenShell supervisor semantics instead # of treating that mutable display metadata as task replacement. Start - # time and procfs identity still bind the original task and fail closed on - # a real replacement. + # time and procfs identity still bind the original task. The executable + # identity rejects an exec replacement that forges the expected argv0. if ( process is None or not _process_has_kernel_task_reference(process, reference) + or not _process_has_reference_executable(process, reference) or not _is_openshell_supervisor(process) ): _fail("supervisor-identity-drift") @@ -2776,9 +2817,13 @@ def _signal_exact_process(process: ProcessIdentity, requested_signal: int) -> No # The full process reference was authenticated before opening the # pidfd. Recheck only immutable kernel task identity here: parent, # credentials, and argv can change on the same task between the two - # observations. The pidfd remains bound to that task, while a real PID - # replacement changes its start time or procfs inode and fails closed. - if current.kernel_task_key() != process.kernel_task_key(): + # observations. The pidfd remains bound to that task. A real PID + # replacement changes its start time or procfs inode, while an exec + # replacement changes its executable identity; both fail closed. + if ( + current.kernel_task_key() != process.kernel_task_key() + or current.executable_key() != process.executable_key() + ): _fail("writer-pid-reused") try: signal.pidfd_send_signal(pidfd, requested_signal) diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index 3df202ff6da..a08601bb8d9 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -54,6 +54,8 @@ "commandSha256", "procDevice", "procInode", + "executableDevice", + "executableInode", ) PERMIT_KEYS = ( "schemaVersion", @@ -172,6 +174,10 @@ def _stable_stat(value: os.stat_result) -> tuple[object, ...]: ) +def _same_filesystem_object(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + def _read_at( directory_fd: int, name: str, @@ -318,12 +324,15 @@ def _parse_uids(raw: bytes) -> tuple[int, int, int, int]: def _capture_parent() -> dict[str, object]: pid = os.getppid() process_path = f"/proc/{pid}" + executable_path = f"{process_path}/exe" try: before = os.stat(process_path, follow_symlinks=False) + executable_before = os.stat(executable_path) first = _read_proc_file(f"{process_path}/stat") status = _read_proc_file(f"{process_path}/status") command_raw = _read_proc_file(f"{process_path}/cmdline") second = _read_proc_file(f"{process_path}/stat") + executable_after = os.stat(executable_path) after = os.stat(process_path, follow_symlinks=False) except OSError: _fail("start-process-unavailable") @@ -332,6 +341,7 @@ def _capture_parent() -> dict[str, object]: if ( not stat.S_ISDIR(before.st_mode) or _stable_stat(before) != _stable_stat(after) + or not _same_filesystem_object(executable_before, executable_after) or (first_parent, first_start) != (second_parent, second_start) ): _fail("start-process-unavailable") @@ -344,6 +354,8 @@ def _capture_parent() -> dict[str, object]: "commandSha256": _command_sha256(command), "procDevice": str(before.st_dev), "procInode": str(before.st_ino), + "executableDevice": str(executable_after.st_dev), + "executableInode": str(executable_after.st_ino), } @@ -368,6 +380,8 @@ def _process_reference(value: object, code: str) -> dict[str, object]: "commandSha256": _hex(value["commandSha256"], code), "procDevice": _decimal(value["procDevice"], code), "procInode": _decimal(value["procInode"], code), + "executableDevice": _decimal(value["executableDevice"], code), + "executableInode": _decimal(value["executableInode"], code), } diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index e3931d84e72..a5f57cb1266 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -170,7 +170,9 @@ def status_value(action, acquire, provider_handle=None, activation_handle=None, def parse(action, value): return control._parse_request(action, control._json_bytes(value) + b"\n") -def process(pid, state, parent, start, uid, command, inode): +def process(pid, state, parent, start, uid, command, inode, executable_inode=None): + if executable_inode is None: + executable_inode = 10_000 + inode return control.ProcessIdentity( pid, state, @@ -180,6 +182,8 @@ def process(pid, state, parent, start, uid, command, inode): command, 91, inode, + 81, + executable_inode, ) root_uid = control.ROOT_UID @@ -282,6 +286,16 @@ replacement_task = process( start.command, start.proc_inode + 1, ) +same_task_executable_replacement = process( + start.pid, + start.state, + start.parent_pid, + start.start_identity, + 1001, + start.command, + start.proc_inode, + start.executable_inode + 1, +) real_pidfd_open = getattr(control.os, "pidfd_open", None) real_pidfd_send_signal = getattr(control.signal, "pidfd_send_signal", None) real_os_close = control.os.close @@ -305,6 +319,14 @@ try: lambda: real_signal_exact_process(start, signal.SIGSTOP) ) results["replacement_task_signal_events"] = list(pidfd_signal_events) + pidfd_signal_events.clear() + control._capture_process = lambda _pid: same_task_executable_replacement + results["same_task_executable_replacement_signal"] = code( + lambda: real_signal_exact_process(start, signal.SIGSTOP) + ) + results["same_task_executable_replacement_signal_events"] = list( + pidfd_signal_events + ) finally: control._capture_process = real_capture_process if real_pidfd_open is None: @@ -728,8 +750,19 @@ replaced_pid1 = process( (control.OPENSHELL_ARGV0,), 102, ) +exec_replaced_pid1 = process( + 1, + "T", + 0, + pid1.start_identity, + root_uid, + (control.OPENSHELL_ARGV0, b"--forged-status"), + pid1.proc_inode, + pid1.executable_inode + 1, +) results["refreshed_supervisor"] = supervisor_refresh(refreshed_pid1) results["replaced_supervisor"] = supervisor_refresh(replaced_pid1) +results["exec_replaced_supervisor"] = supervisor_refresh(exec_replaced_pid1) hold_events = [] control._prove_fence_shape = lambda _fence, _mount: (pid1, start) @@ -1181,6 +1214,8 @@ def state_process(original): original.command, original.proc_device, original.proc_inode, + original.executable_device, + original.executable_inode, ) by_release_pid = { 1: refreshed_pid1, diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 99c5bbcd811..77853d62ba9 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -54,7 +54,12 @@ describe("runtime state mutation controller", () => { bare_direct_start: false, bare_interpreted_start: false, acquire_fence: { - supervisor: { pid: 1, startIdentity: "100" }, + supervisor: { + pid: 1, + startIdentity: "100", + executableDevice: "81", + executableInode: "10101", + }, start: { pid: 10, startIdentity: "200", parentPid: 1 }, startSupport: [ { pid: 75, parentPid: 10 }, @@ -129,11 +134,17 @@ describe("runtime state mutation controller", () => { ["open", 10, 0], ["close", 91], ]); + expect(harnessResult.same_task_executable_replacement_signal).toBe("writer-pid-reused"); + expect(harnessResult.same_task_executable_replacement_signal_events).toEqual([ + ["open", 10, 0], + ["close", 91], + ]); }); - it("retains the same OpenShell supervisor across mutable argv metadata (#9485)", () => { + it("retains supervisor argv refreshes but rejects same-task executable replacement (#9485)", () => { expect(harnessResult.refreshed_supervisor).toBe("ok"); expect(harnessResult.replaced_supervisor).toBe("supervisor-identity-drift"); + expect(harnessResult.exec_replaced_supervisor).toBe("supervisor-identity-drift"); }); it("publishes, rolls an activated fence back, and recovers every durable phase (#7744)", () => { diff --git a/test/state/runtime-state-mutation-release-ack.test.ts b/test/state/runtime-state-mutation-release-ack.test.ts index 9165753d6a5..72caa383204 100644 --- a/test/state/runtime-state-mutation-release-ack.test.ts +++ b/test/state/runtime-state-mutation-release-ack.test.ts @@ -37,6 +37,8 @@ start = control.ProcessReference( "a" * 64, 12, 13, + 14, + 15, ) fence = control.FenceProof(start, start, (), (os.geteuid(),)) marker = {"transactionId": "c" * 64, "nonce": "b" * 64} @@ -100,6 +102,8 @@ with tempfile.TemporaryDirectory() as root: "commandSha256": "a" * 64, "procDevice": "12", "procInode": "13", + "executableDevice": "14", + "executableInode": "15", }, } assert expected == control._startup_release_ack_payload( @@ -144,6 +148,8 @@ with tempfile.TemporaryDirectory() as root: ), 14, 15, + 16, + 17, ) publisher_scans = {"count": 0} publisher_signals = [] @@ -164,6 +170,8 @@ with tempfile.TemporaryDirectory() as root: (control.NEMOCLAW_START_PATH,), start.proc_device, start.proc_inode, + start.executable_device, + start.executable_inode, ) control._recapture_reference = recapture def signal_publisher(selected, requested): diff --git a/test/state/runtime-state-mutation-startup-gate.test.ts b/test/state/runtime-state-mutation-startup-gate.test.ts index 2e2fdd7ab4e..acfc99f1bd9 100644 --- a/test/state/runtime-state-mutation-startup-gate.test.ts +++ b/test/state/runtime-state-mutation-startup-gate.test.ts @@ -37,6 +37,8 @@ start = { "commandSha256": "a" * 64, "procDevice": "22", "procInode": "33", + "executableDevice": "44", + "executableInode": "55", } gate._capture_parent = lambda: start @@ -220,6 +222,8 @@ describe("runtime state mutation startup gate", () => { commandSha256: "a".repeat(64), procDevice: "22", procInode: "33", + executableDevice: "44", + executableInode: "55", }; expect(value).toMatchObject({ uses_o_path_when_available: true, From a4242356331e981bf4dcbd33d2e03daf156e65d3 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 15:19:23 -0700 Subject: [PATCH 39/44] fix(runtime): distinguish invalid startup gate state --- agents/hermes/start.sh | 5 + docs/reference/troubleshooting.mdx | 5 + .../runtime-state-mutation-startup-gate.py | 109 ++++++++++++------ ...me-state-mutation-hermes-publisher.test.ts | 32 ++++- ...untime-state-mutation-startup-gate.test.ts | 25 ++++ 5 files changed, 138 insertions(+), 38 deletions(-) diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index a54367c6330..fa4490fa61d 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -80,6 +80,11 @@ while :; do printf '%s\n' '[SECURITY] Hermes startup held by an active runtime state mutation.' >&2 /bin/sleep 1 || true ;; + 76) + printf '%s\n' '[SECURITY] Hermes startup refused invalid runtime state mutation state.' >&2 + printf '%s\n' "[SECURITY] Run 'nemoclaw shields status' on the host to recover the retained transition." >&2 + exit 1 + ;; *) printf '%s\n' '[SECURITY] Runtime state mutation startup gate failed.' >&2 exit 1 diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index ebd28ff636c..422ba625ffe 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1834,6 +1834,11 @@ Do not kill or manually restart the held entrypoint. Run `$$nemoclaw shields status` from the host so NemoClaw can recover the retained target and authenticate the startup release. If recovery still fails, preserve the owner-only lifecycle ledger and the complete error for diagnosis instead of removing a marker or changing in-sandbox permissions. +If the startup gate instead reports `invalid-state code= transaction=` and Hermes refuses startup, the fixed code identifies rejected gate state rather than an active transition that can make progress by waiting. +The transaction is a 64-character identifier when the gate safely parsed it, or `unknown` when it could not. +Run `$$nemoclaw shields status` from the host to recover the retained transition. +If status cannot recover it, preserve `~/.nemoclaw/state/runtime-provider-lifecycle/`, the fixed code, and the transaction identifier for diagnosis; do not remove or edit the root-owned in-sandbox receipts. + ### Hermes startup reports `HERMES_CONFIG_MUTATION_ORPHANED` This refusal means a root-owned config or shields transaction stopped without a safely recoverable complete state. diff --git a/scripts/runtime-state-mutation-startup-gate.py b/scripts/runtime-state-mutation-startup-gate.py index a08601bb8d9..5f4466de85e 100755 --- a/scripts/runtime-state-mutation-startup-gate.py +++ b/scripts/runtime-state-mutation-startup-gate.py @@ -90,6 +90,16 @@ class GateError(RuntimeError): """A fixed, non-sensitive startup-gate refusal.""" + def __init__(self, code: str, transaction_id: str | None = None) -> None: + super().__init__(code) + self.code = code + self.transaction_id = transaction_id + + def with_transaction(self, transaction_id: str) -> GateError: + if self.transaction_id is not None: + return self + return GateError(self.code, transaction_id) + def _fail(code: str) -> NoReturn: raise GateError(code) @@ -392,41 +402,46 @@ def _binding(raw: bytes, protocol: str, keys: tuple[str, ...]) -> dict[str, obje if value["schemaVersion"] != SCHEMA_VERSION or value["protocol"] != protocol: _fail("gate-receipt-invalid") transaction_id = _hex(value["transactionId"], "gate-receipt-invalid") - nonce = _hex(value["nonce"], "gate-receipt-invalid") - expected_directory = f"{HANDOFF_ROOT}/{nonce}" - if value["candidateDirectory"] != expected_directory: - _fail("gate-receipt-invalid") - start = _process_reference(value["start"], "gate-receipt-invalid") - if _canonical(start) != _canonical(_capture_parent()): - _fail("gate-start-mismatch") - if protocol == PERMIT_PROTOCOL: - protocol_binding: dict[str, object] = { - "markerSha256": _hex(value["markerSha256"], "gate-receipt-invalid") - } - elif protocol == RELEASE_PROTOCOL: - protocol_binding = { - "checkpointSha256": _hex(value["checkpointSha256"], "gate-receipt-invalid") - } - else: - checkpoint = value["checkpointSha256"] - protocol_binding = { - "permitSha256": _hex(value["permitSha256"], "gate-receipt-invalid"), - "checkpointSha256": ( - None if checkpoint is None else _hex(checkpoint, "gate-receipt-invalid") - ), + try: + nonce = _hex(value["nonce"], "gate-receipt-invalid") + expected_directory = f"{HANDOFF_ROOT}/{nonce}" + if value["candidateDirectory"] != expected_directory: + _fail("gate-receipt-invalid") + start = _process_reference(value["start"], "gate-receipt-invalid") + if _canonical(start) != _canonical(_capture_parent()): + _fail("gate-start-mismatch") + if protocol == PERMIT_PROTOCOL: + protocol_binding: dict[str, object] = { + "markerSha256": _hex(value["markerSha256"], "gate-receipt-invalid") + } + elif protocol == RELEASE_PROTOCOL: + protocol_binding = { + "checkpointSha256": _hex(value["checkpointSha256"], "gate-receipt-invalid") + } + else: + checkpoint = value["checkpointSha256"] + protocol_binding = { + "permitSha256": _hex(value["permitSha256"], "gate-receipt-invalid"), + "checkpointSha256": ( + None + if checkpoint is None + else _hex(checkpoint, "gate-receipt-invalid") + ), + } + normalized = { + "schemaVersion": SCHEMA_VERSION, + "protocol": protocol, + "transactionId": transaction_id, + "nonce": nonce, + **protocol_binding, + "start": start, + "candidateDirectory": expected_directory, } - normalized = { - "schemaVersion": SCHEMA_VERSION, - "protocol": protocol, - "transactionId": transaction_id, - "nonce": nonce, - **protocol_binding, - "start": start, - "candidateDirectory": expected_directory, - } - if raw != _canonical(normalized) + b"\n": - _fail("gate-receipt-invalid") - return normalized + if raw != _canonical(normalized) + b"\n": + _fail("gate-receipt-invalid") + return normalized + except GateError as error: + raise error.with_transaction(transaction_id) from None def _read_binding( @@ -708,17 +723,20 @@ def _run(action: str) -> str: _fail("activation-release-missing") return "inactive" assert directory_fd is not None + transaction_id: str | None = None try: released = _read_binding( directory_fd, RELEASE_NAME, RELEASE_PROTOCOL, RELEASE_KEYS ) if released is not None: + transaction_id = str(released["transactionId"]) _verify_release_candidate(released) if action == "acknowledge": return _prepare_release_ack(released) return "released" retry = _read_binding(directory_fd, RETRY_NAME, RETRY_PROTOCOL, RETRY_KEYS) if retry is not None: + transaction_id = str(retry["transactionId"]) retry_payload = _verify_retry_binding(directory_fd, retry) if action == "admit": _publish_retry_ack(retry, retry_payload) @@ -729,12 +747,17 @@ def _run(action: str) -> str: permit = _read_binding(directory_fd, PERMIT_NAME, PERMIT_PROTOCOL, PERMIT_KEYS) if permit is None: _fail("activation-not-permitted") + transaction_id = str(permit["transactionId"]) if action == "checkpoint": _publish_candidate(permit) return "activation-ready" if action in ("restart", "resume"): _fail("activation-not-released") return "permitted" + except GateError as error: + if transaction_id is None: + raise + raise error.with_transaction(transaction_id) from None finally: os.close(directory_fd) @@ -767,9 +790,21 @@ def main(argv: list[str] | None = None) -> int: "retry": 12, "retry-wait": 75, }[state] - except (GateError, OSError): - print("runtime-state-mutation-startup-gate: held", file=sys.stderr) - return 75 + except GateError as error: + transaction_id = error.transaction_id or "unknown" + print( + "runtime-state-mutation-startup-gate: invalid-state " + f"code={error.code} transaction={transaction_id}", + file=sys.stderr, + ) + return 76 + except OSError: + print( + "runtime-state-mutation-startup-gate: invalid-state " + "code=gate-io-error transaction=unknown", + file=sys.stderr, + ) + return 76 if __name__ == "__main__": diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index 666404e1976..7e37237cb95 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -353,6 +353,10 @@ function createEntrypointGateFixture(temporary: string): EntrypointGateFixture { " admit)", ' printf "admit\\n" >> "$NEMOCLAW_TEST_TRACE"', ' if [ "$NEMOCLAW_TEST_GATE_MODE" = "deny" ]; then exit 1; fi', + ' if [ "$NEMOCLAW_TEST_GATE_MODE" = "invalid" ]; then', + ' printf "%s\\n" "runtime-state-mutation-startup-gate: invalid-state code=gate-receipt-invalid transaction=${NEMOCLAW_TEST_TRANSACTION}" >&2', + " exit 76", + " fi", " exit 10", " ;;", " checkpoint)", @@ -446,7 +450,7 @@ async function waitForStoppedProcess(pid: number, description: string): Promise< function entrypointGateEnvironment( fixture: EntrypointGateFixture, - gateMode: "allow" | "deny", + gateMode: "allow" | "deny" | "invalid", ): NodeJS.ProcessEnv { return { ...process.env, @@ -456,6 +460,7 @@ function entrypointGateEnvironment( NEMOCLAW_TEST_GATE_MODE: gateMode, NEMOCLAW_TEST_RELEASE: fixture.releasePath, NEMOCLAW_TEST_TRACE: fixture.tracePath, + NEMOCLAW_TEST_TRANSACTION: "c".repeat(64), }; } @@ -635,4 +640,29 @@ describe("Hermes runtime state mutation publisher", () => { fs.rmSync(temporary, { force: true, recursive: true }); } }); + + it("fails closed with stable host recovery guidance for invalid gate state (#10155)", () => { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-entrypoint-invalid-")); + const fixture = createEntrypointGateFixture(temporary); + try { + const result = spawnSync("bash", [fixture.harnessPath], { + encoding: "utf8", + env: entrypointGateEnvironment(fixture, "invalid"), + timeout: 5_000, + }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "invalid-state code=gate-receipt-invalid transaction=" + "c".repeat(64), + ); + expect(result.stderr).toContain( + "Run 'nemoclaw shields status' on the host", + ); + expect(result.stderr).not.toContain("held by an active runtime state mutation"); + expect(readEntrypointTrace(fixture.tracePath)).toEqual(["admit"]); + expect(fs.existsSync(fixture.acknowledgePath)).toBe(false); + } finally { + fs.rmSync(temporary, { force: true, recursive: true }); + } + }); }); diff --git a/test/state/runtime-state-mutation-startup-gate.test.ts b/test/state/runtime-state-mutation-startup-gate.test.ts index acfc99f1bd9..e76adbe1476 100644 --- a/test/state/runtime-state-mutation-startup-gate.test.ts +++ b/test/state/runtime-state-mutation-startup-gate.test.ts @@ -16,11 +16,13 @@ const GATE = path.join( const HARNESS = String.raw` import hashlib import importlib.util +import io import json import os import signal import sys import tempfile +from contextlib import redirect_stderr spec = importlib.util.spec_from_file_location("runtime_state_startup_gate", sys.argv[1]) gate = importlib.util.module_from_spec(spec) @@ -95,6 +97,21 @@ with tempfile.TemporaryDirectory() as root: } write(os.path.join(durable, gate.PERMIT_NAME), permit, 0o444) results["admitted"] = gate._run("admit") + gate._capture_parent = lambda: {**start, "pid": 42} + invalid_stderr = io.StringIO() + with redirect_stderr(invalid_stderr): + results["invalid_status"] = gate.main(["admit"]) + results["invalid_stderr"] = invalid_stderr.getvalue().strip() + gate._capture_parent = lambda: start + os.chmod(os.path.join(durable, gate.PERMIT_NAME), 0o600) + with open(os.path.join(durable, gate.PERMIT_NAME), "wb") as stream: + stream.write(b"{\n") + os.chmod(os.path.join(durable, gate.PERMIT_NAME), 0o444) + malformed_stderr = io.StringIO() + with redirect_stderr(malformed_stderr): + results["malformed_status"] = gate.main(["admit"]) + results["malformed_stderr"] = malformed_stderr.getvalue().strip() + write(os.path.join(durable, gate.PERMIT_NAME), permit, 0o444) orphan = os.path.join(candidate_directory, ".startup-complete.json.91.interrupted") with open(orphan, "wb") as stream: @@ -248,6 +265,14 @@ describe("runtime state mutation startup gate", () => { tampered_release: "release-candidate-mismatch", symlink_directory: "unsafe-directory", invalid_present_directory: "gate-directory-invalid", + invalid_status: 76, + invalid_stderr: + "runtime-state-mutation-startup-gate: invalid-state " + + `code=gate-start-mismatch transaction=${"c".repeat(64)}`, + malformed_status: 76, + malformed_stderr: + "runtime-state-mutation-startup-gate: invalid-state " + + "code=gate-receipt-invalid transaction=unknown", candidate: { schemaVersion: 1, protocol: "nemoclaw-runtime-state-mutation-startup-complete-v1", From 7649bfbbb9a6da037607c3a8c28668dc2b4e90ac Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 16:05:42 -0700 Subject: [PATCH 40/44] refactor(runtime): install fixed transport broker --- agents/hermes/Dockerfile | 8 +- docs/reference/commands.mdx | 2 +- docs/security/tcb-boundary.mdx | 1 + scripts/runtime-state-mutation-control.py | 14 +- ...runtime-state-mutation-transport-broker.py | 500 ++++++++++++++++++ .../hermes-portable-build-context-files.ts | 1 + .../hermes-portable-build-context.ts | 1 + .../docker-state-mutation.test.ts | 55 +- .../runtime-provider/docker-state-mutation.ts | 326 +----------- .../hermes/hermes-doctor-config-hash.test.ts | 9 +- .../runtime-state-mutation-control-harness.ts | 40 +- .../sandbox/sandbox-provisioning.test.ts | 10 +- .../runtime-state-mutation-control.test.ts | 2 + 13 files changed, 617 insertions(+), 352 deletions(-) create mode 100755 scripts/runtime-state-mutation-transport-broker.py diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index badb842c367..e513693852c 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -130,6 +130,7 @@ COPY agents/hermes/cron-restore-control.py /usr/local/lib/nemoclaw/hermes-cron-r COPY src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.106.json /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.106.json COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py COPY scripts/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py +COPY scripts/runtime-state-mutation-transport-broker.py /usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py COPY scripts/runtime-state-mutation-startup-gate.py /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py COPY scripts/runtime_state_mutation_hermes_publisher.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py COPY agents/hermes/state-lock-plan.json /usr/local/share/nemoclaw/state-lock-plan.json @@ -493,14 +494,14 @@ RUN chmod -R a+rX /opt/nemoclaw-blueprint/ # minimum supported Hermes sandbox base tag guarantees those artifacts and # test/runtime/sandbox/sandbox-rlimit-hooks.test.ts covers that base. RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-managed-startup-hold /usr/local/bin/nemoclaw-managed-bootstrap /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/patch-hermes-session-list-preview.py /usr/local/lib/nemoclaw/patch-hermes-sqlite-temp-store.py /usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py /usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py /usr/local/lib/nemoclaw/finalize-tirith-marker.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py \ - && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py /usr/local/share/nemoclaw/state-lock-plan.json /usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/hermes-cron-restore-control.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.106.json \ + && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py /usr/local/share/nemoclaw/state-lock-plan.json /usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/hermes-cron-restore-control.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.106.json \ && chmod 700 /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/hermes-cron-restore-control.py \ - && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py /usr/local/lib/nemoclaw/managed-gateway-control.py \ + && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/runtime-state-mutation-control.py /usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py /usr/local/lib/nemoclaw/managed-gateway-control.py \ && chmod 555 /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py \ && chmod 444 /usr/local/lib/nemoclaw/corporate-ca-runtime.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh /usr/local/share/nemoclaw/state-lock-plan.json /usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/managed_policy.py \ && chmod 444 /usr/local/lib/nemoclaw/patch-hermes-langfuse-credentials.mts \ && chmod 444 /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.106.json \ - && /opt/hermes/.venv/bin/python3 -I -c 'import runpy, yaml; assert yaml.safe_load("ready: true")["ready"] is True; runpy.run_path("/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", run_name="nemoclaw_runtime_state_mutation_control_probe"); runpy.run_path("/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py", run_name="nemoclaw_runtime_state_mutation_publisher_probe"); runpy.run_path("/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py", run_name="nemoclaw_runtime_state_mutation_gate_probe"); runpy.run_path("/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", run_name="nemoclaw_runtime_config_guard_probe")' \ + && /opt/hermes/.venv/bin/python3 -I -c 'import runpy, yaml; assert yaml.safe_load("ready: true")["ready"] is True; runpy.run_path("/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", run_name="nemoclaw_runtime_state_mutation_control_probe"); runpy.run_path("/usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py", run_name="nemoclaw_runtime_state_mutation_transport_broker_probe"); runpy.run_path("/usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py", run_name="nemoclaw_runtime_state_mutation_publisher_probe"); runpy.run_path("/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py", run_name="nemoclaw_runtime_state_mutation_gate_probe"); runpy.run_path("/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", run_name="nemoclaw_runtime_config_guard_probe")' \ && if [ -d /usr/local/lib/nemoclaw/preloads ]; then \ chown -R 0:0 /usr/local/lib/nemoclaw/preloads \ && find /usr/local/lib/nemoclaw/preloads -type f -exec chmod 444 {} + \ @@ -1450,6 +1451,7 @@ RUN check_metadata() { \ && check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root 700' \ && check_metadata /usr/local/share/nemoclaw/state-lock-plan.json 'root:root 444' \ && check_metadata /usr/local/lib/nemoclaw/runtime-state-mutation-control.py 'root:root 500' \ + && check_metadata /usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py 'root:root 500' \ && check_metadata /usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py 'root:root 555' \ && check_metadata /usr/local/lib/nemoclaw/runtime_state_mutation_hermes_publisher.py 'root:root 500' \ && check_metadata /var/lib/nemoclaw/runtime-state-mutation 'root:root 711' \ diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 994dacada4d..44001583900 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -5560,7 +5560,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | -| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive whole-number milliseconds (default `750`, maximum `10000`) | NemoClaw waits this long after re-applying a config lockdown before checking that the lock still holds. It applies during ordinary `$$nemoclaw shields up` transitions, shields auto-restore, and `shields up` drift remediation. Fractional, zero, negative, blank, and invalid values use the default. If NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This check narrows the window in which an in-sandbox reconciler can revert permissions; it does not eliminate that window. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise the value on hosts where the gateway settles slowly. | +| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive whole-number milliseconds (default `750`, maximum `10000`) | NemoClaw waits this long after re-applying a config lockdown before checking that the lock still holds. It applies during ordinary `$$nemoclaw shields up` transitions, shields auto-restore, and `shields up` drift remediation. Values above `10000` use `10000`. Fractional, zero, negative, blank, and invalid values use the default. If NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This check narrows the window in which an in-sandbox reconciler can revert permissions; it does not eliminate that window. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise the value on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | | `NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS` | `1` to opt in | Makes `$$nemoclaw uninstall` remove every gateway port on the host instead of only the port `NEMOCLAW_GATEWAY_PORT` selects. Equivalent to passing the `--all-gateway-ports` flag; the whole-host `Proceed?` confirmation still applies unless `--yes` is also passed. Each port runs as its own uninstall, and the variable is dropped from those runs so the sweep cannot re-enter itself. | | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA` | `1` to opt in | Acknowledges data loss during `$$nemoclaw uninstall`, skips eligible fresh sandbox backups, and removes the otherwise-preserved entries (`rebuild-backups/`, `backups/`, `sandboxes.json`) in the selected gateway's state root. It does not select the explicit `--destroy-user-data` CLI-shim removal path; shim handling follows the ordinary selected-gateway scope. The global `Proceed?` confirmation still applies unless `--yes` is also passed. | diff --git a/docs/security/tcb-boundary.mdx b/docs/security/tcb-boundary.mdx index 0b7ba49bb55..fd3a056ced3 100644 --- a/docs/security/tcb-boundary.mdx +++ b/docs/security/tcb-boundary.mdx @@ -56,6 +56,7 @@ A successful build does not replace review of privilege, process identity, descr | `src/lib/onboard/runtime-provider/docker-operation-authority.ts` | Runs in the host CLI under the operator account and invokes Docker without a shell from a fixed working directory and sanitized environment. | One qualified absolute Docker executable and its absolute interpreter chain; an absolute-only fixed `PATH`; the effective PATH-selected `docker-credential-*` executables and their interpreter chains; the effective SSH executable for an `ssh://` endpoint; the context, host, TLS, and endpoint bindings; and the non-`PATH` execution-environment digest. The complete `PATH` enters authority identity for host-local inference and `ssh://`. A local `sandbox-lifecycle` authority omits helper-free added directories from persisted identity while still binding selected delegated helpers. | Revalidates executable, interpreter, delegated-command, and engine-endpoint metadata before provider actions; rejects authority drift and unverified remote TCP; and prevents ambient Docker configuration or command delegation from redirecting a fenced runtime provider state mutation. | | `src/lib/onboard/runtime-provider/persisted-engine-lifecycle.ts` and `~/.nemoclaw/state/runtime-provider-lifecycle/` | The host CLI manages an owner-only private directory and durable artifacts under the operator account. | A runtime target claim and phase-bound transaction, release receipt, or tombstone whose sandbox lifecycle, engine binding, container, state root, plan, projection, target, rollback, and nonce match. | Preserves recovery authority across host process restart, excludes a second target, advances only the transaction through `prepared`, `mutation-authorized`, `fence-established`, and `completed` phases, and releases only the matching claim. | | `src/lib/sandbox/privileged-exec.ts` and `src/lib/adapters/sandbox/command-transport.ts` | Run in the host CLI and hold a privileged sandbox execution lease through command completion and transport cleanup. | The canonical per-sandbox host transition lock, the durable runtime target claim, and the registered container identity. | Drains an earlier execution lease before provider fencing and rejects later direct-container, OpenShell, and SSH execution before argument resolution or process spawn. One OpenShell lease covers its direct Docker fallback so transport selection cannot cross the fence. | +| `scripts/runtime-state-mutation-transport-broker.py` | The current managed Hermes image installs a root-owned, mode `0500` copy. The qualified Docker provider starts its fixed path as root before the supervisor stops. | One 64-character transaction identifier, fixed controller and transport paths, root-owned private request files, and bounded action-specific timeouts. | Keeps the fixed controller reachable while ordinary command transports are fenced, validates transaction-bound requests and acknowledgements, normalizes helper failures, and removes its private session only after the released supervisor is confirmed running. | | `scripts/runtime-state-mutation-control.py` | The current managed Hermes image installs a root-owned, mode `0500` copy, invoked as root through the qualified Docker provider authority. | Fixed actions; container, engine, mount-namespace, state-root, plan, projection, posture, and nonce bindings; installed controller and publisher identities; and stable process evidence. | Establishes and verifies the in-container fence, stops the bound entrypoint and sandbox processes, invokes only the fixed Hermes publisher, starts only the bound entrypoint, requires a fresh healthy gateway and authenticated startup checkpoint, and retains restrictive state when release is not proved. | | `scripts/runtime_state_mutation_hermes_publisher.py` | The current managed Hermes image installs a root-owned, mode `0500` copy that only the runtime provider state mutation controller imports. | The root-owned controller marker, nonce-bound canonical plan and projection, installed state-lock plan, state-root descriptor, and requested posture. | Applies the normalized Hermes recursive posture, publishes fresh protected inodes where required, maintains its root-only journal, verifies the result, and returns a bounded receipt to the controller. | | `scripts/runtime-state-mutation-startup-gate.py` and `agents/hermes/start.sh` | The gate is root-owned and mode `0555`; the Hermes entrypoint invokes it before sourcing helpers or reading mutable state. | The parent process, root-owned controller handoff, active runtime provider state mutation identity, candidate checkpoint, and authenticated release for the same target. | Holds or refuses startup on active, malformed, uninspectable, or unauthenticated state, publishes a checkpoint only after the complete Hermes topology is healthy, and resumes only the controller-authenticated candidate. | diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index d50df81db5d..595c6d677cd 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -117,11 +117,9 @@ START_LOG_DRAIN_PATHS = (b"tee", b"/usr/bin/tee", b"/bin/tee") STARTUP_GATE_PYTHON = b"/opt/hermes/.venv/bin/python3" STARTUP_GATE_HELPER = b"/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py" -TRANSPORT_BROKER_BOOTSTRAP = ( - b"import base64,sys,zlib;source=zlib.decompress(base64.b64decode(sys.argv.pop(1)),-15);" - b"exec(compile(source,'','exec'))" +TRANSPORT_BROKER_PATH = ( + b"/usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py" ) -TRANSPORT_BROKER_HELPER_PATH = b"/usr/local/lib/nemoclaw/runtime-state-mutation-control.py" HEX_64 = re.compile(r"[0-9a-f]{64}\Z") SAFE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z") @@ -3190,12 +3188,10 @@ def _transport_broker_reference() -> ProcessReference | None: process.pid <= 1 or process.state in ("Z", "X", "x") or process.uids != (ROOT_UID,) * 4 - or len(command) != 7 + or len(command) != 4 or command[1] != b"-I" - or command[2] != b"-c" - or command[3] != TRANSPORT_BROKER_BOOTSTRAP - or command[5] != TRANSPORT_BROKER_HELPER_PATH - or re.fullmatch(rb"[0-9a-f]{64}", command[6]) is None + or command[2] != TRANSPORT_BROKER_PATH + or re.fullmatch(rb"[0-9a-f]{64}", command[3]) is None ): return None return _process_reference(process) diff --git a/scripts/runtime-state-mutation-transport-broker.py b/scripts/runtime-state-mutation-transport-broker.py new file mode 100755 index 00000000000..b8dda952187 --- /dev/null +++ b/scripts/runtime-state-mutation-transport-broker.py @@ -0,0 +1,500 @@ +#!/opt/hermes/.venv/bin/python3 -I +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Broker fixed root state-mutation requests while the supervisor is stopped.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import re +import signal +import stat +import subprocess +import sys +import time + + +ROOT = "/run/nemoclaw/runtime-state-mutation" +HELPER = "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py" +EXPECTED_UID = 0 +EXPECTED_GID = 0 +MAXIMUM = 128 * 1024 +TIMEOUTS = { + "acquire": 30, + "assert": 30, + "publish": 900, + "recover": 900, + "rollback": 900, + "activate": 485, + "release": 300, +} +IDENTITY = re.compile(r"[a-f0-9]{64}\Z") +INCOMING = re.compile( + r"([a-f0-9]{64})\.(acquire|assert|publish|recover|rollback|activate|release)\.incoming\Z" +) +PUBLICATION_SETTLE_SECONDS = 5 + + +def fail(code: str) -> None: + raise RuntimeError(code) + + +def directory(path: str) -> None: + metadata = os.lstat(path) + if ( + not stat.S_ISDIR(metadata.st_mode) + or metadata.st_uid != EXPECTED_UID + or metadata.st_gid != EXPECTED_GID + or stat.S_IMODE(metadata.st_mode) != 0o700 + ): + fail("transport-directory-invalid") + + +def atomic(path: str, payload: bytes) -> None: + temporary = path + ".tmp-" + str(os.getpid()) + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + 0o600, + ) + try: + offset = 0 + while offset < len(payload): + written = os.write(descriptor, payload[offset:]) + if written <= 0: + fail("transport-write-failed") + offset += written + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, path) + + +def private_file(path: str) -> bytes: + descriptor = os.open( + path, + os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK, + ) + try: + before = os.fstat(descriptor) + payload = os.read(descriptor, MAXIMUM + 1) + after = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_uid != EXPECTED_UID + or before.st_gid != EXPECTED_GID + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_nlink != 1 + or len(payload) > MAXIMUM + or os.read(descriptor, 1) + or ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_nlink, + before.st_uid, + before.st_gid, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + != ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_nlink, + after.st_uid, + after.st_gid, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + ): + fail("transport-file-invalid") + return payload + finally: + os.close(descriptor) + + +def copied_file(path: str) -> bytes: + descriptor = os.open( + path, + os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK, + ) + try: + before = os.fstat(descriptor) + payload = bytearray() + while len(payload) <= MAXIMUM: + chunk = os.read( + descriptor, + min(64 * 1024, MAXIMUM + 1 - len(payload)), + ) + if not chunk: + break + payload.extend(chunk) + after = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or len(payload) > MAXIMUM + or ( + before.st_dev, + before.st_ino, + before.st_nlink, + before.st_uid, + before.st_gid, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + != ( + after.st_dev, + after.st_ino, + after.st_nlink, + after.st_uid, + after.st_gid, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + ): + fail("transport-copied-file-invalid") + return bytes(payload) + finally: + os.close(descriptor) + + +def response_payload( + action: str, + identity: str, + status_code: int, + stdout: str, + stderr: str, +) -> bytes: + return ( + json.dumps( + { + "schemaVersion": 1, + "action": action, + "identity": identity, + "status": status_code, + "stdout": stdout, + "stderr": stderr, + }, + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + + +def failure_stderr(action: str, code: str) -> str: + return ( + json.dumps( + { + "schemaVersion": 1, + "action": action, + "status": "failed", + "code": code, + }, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ) + + +def post_validation_failure_code(error: BaseException) -> str: + if isinstance(error, RuntimeError): + code = str(error) + if code in ( + "helper-file-missing", + "helper-file-invalid", + "transport-response-too-large", + ): + return code + return "transport-runtime-failed" + if isinstance(error, UnicodeError): + return "transport-response-encoding-invalid" + if isinstance(error, FileNotFoundError): + return "transport-resource-missing" + if isinstance(error, PermissionError): + return "transport-permission-denied" + if isinstance(error, OSError): + return "transport-io-failed" + return "transport-response-invalid" + + +def normalize_helper_stderr(action: str, status_code: int, stderr: bytes) -> bytes: + if not stderr: + return stderr + try: + failure = json.loads(stderr.decode("utf-8", "strict")) + if ( + isinstance(failure, dict) + and failure.get("schemaVersion") == 1 + and failure.get("action") == action + and failure.get("status") == "failed" + and isinstance(failure.get("code"), str) + and re.fullmatch(r"[a-z][a-z0-9-]{0,127}", failure["code"]) + is not None + ): + return stderr + except (UnicodeError, ValueError): + pass + code = "helper-process-failed" if status_code != 0 else "helper-protocol-stderr" + return failure_stderr(action, code).encode("utf-8") + + +def publisher_phase_failure(action: str, stderr: bytes) -> bytes: + if action != "publish": + return stderr + try: + failure = json.loads(stderr.decode("utf-8", "strict")) + if ( + not isinstance(failure, dict) + or failure.get("schemaVersion") != 1 + or failure.get("action") != "publish" + or failure.get("status") != "failed" + or failure.get("code") != "publisher-guard-failed" + ): + return stderr + journal = json.loads( + private_file( + "/var/lib/nemoclaw/runtime-state-mutation/hermes-publisher.json" + ).decode("utf-8", "strict") + ) + operation = journal.get("operation") if isinstance(journal, dict) else None + phase = operation.get("phase") if isinstance(operation, dict) else None + if phase not in ("intent", "begun", "state-applied", "top-applied"): + return stderr + failure["code"] = "publisher-guard-" + phase + "-failed" + return ( + json.dumps(failure, ensure_ascii=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + except (OSError, RuntimeError, UnicodeError, ValueError): + return stderr + + +def run_helper(action: str, request: bytes) -> subprocess.CompletedProcess[bytes]: + try: + metadata = os.lstat(HELPER) + except FileNotFoundError: + fail("helper-file-missing") + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != EXPECTED_UID + or metadata.st_gid != EXPECTED_GID + or stat.S_IMODE(metadata.st_mode) & 0o022 + ): + fail("helper-file-invalid") + deadline = time.monotonic() + TIMEOUTS[action] + completed: subprocess.CompletedProcess[bytes] | None = None + for _ in range(2): + process = subprocess.Popen( + [sys.executable, "-I", HELPER, action], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate( + request, + timeout=max(0.001, deadline - time.monotonic()), + ) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except OSError: + try: + process.kill() + except OSError: + pass + try: + process.communicate(timeout=5) + except (subprocess.TimeoutExpired, OSError): + pass + raise + completed = subprocess.CompletedProcess( + process.args, + process.returncode, + stdout=stdout, + stderr=stderr, + ) + if completed.returncode >= 0: + return completed + # Replay one signal exit inside this action's deadline. + assert completed is not None + return completed + + +def serve(transaction: str) -> None: + os.makedirs(ROOT, mode=0o700, exist_ok=True) + directory(ROOT) + session = os.path.join(ROOT, transaction) + os.makedirs(session, mode=0o700, exist_ok=True) + directory(session) + lock = os.open( + os.path.join(session, "broker.lock"), + os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, + 0o600, + ) + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return + atomic(os.path.join(session, "ready"), (transaction + "\n").encode("ascii")) + pending: dict[str, float] = {} + + while True: + names = sorted(os.listdir(session)) + if "released" in names and "resumed" in names: + try: + expected = (transaction + "\n").encode("ascii") + if private_file( + os.path.join(session, "released") + ) == expected and copied_file(os.path.join(session, "resumed")) == expected: + for name in ("released", "resumed", "ready", "broker.lock"): + try: + os.unlink(os.path.join(session, name)) + except FileNotFoundError: + pass + try: + os.rmdir(session) + except OSError: + pass + return + except (OSError, RuntimeError, UnicodeError, ValueError): + pass + for name in names: + incoming = INCOMING.fullmatch(name) + if incoming is None: + continue + identity, action = incoming.groups() + request_path = os.path.join(session, name) + response_path = os.path.join(session, identity + ".response") + if os.path.exists(response_path): + continue + validated = False + try: + request = copied_file(request_path) + if ( + not request.endswith(b"\n") + or hashlib.sha256(request).hexdigest() != identity + ): + fail("transport-request-invalid") + envelope = json.loads(request.decode("utf-8", "strict")) + if ( + not isinstance(envelope, dict) + or envelope.get("action") != action + or envelope.get("transactionId") != transaction + ): + fail("transport-request-invalid") + validated = True + pending.pop(name, None) + os.unlink(request_path) + completed = run_helper(action, request) + if len(completed.stdout) > MAXIMUM or len(completed.stderr) > MAXIMUM: + fail("transport-response-too-large") + status_code = ( + completed.returncode + if completed.returncode >= 0 + else 128 - completed.returncode + ) + stderr = publisher_phase_failure(action, completed.stderr) + stderr = normalize_helper_stderr(action, status_code, stderr) + response = response_payload( + action, + identity, + status_code, + completed.stdout.decode("utf-8", "strict"), + stderr.decode("utf-8", "strict"), + ) + except subprocess.TimeoutExpired: + response = response_payload( + action, + identity, + 1, + "", + failure_stderr(action, "helper-timeout"), + ) + except (OSError, RuntimeError, UnicodeError, ValueError) as error: + if not validated: + first_observed = pending.setdefault(name, time.monotonic()) + if ( + time.monotonic() - first_observed + < PUBLICATION_SETTLE_SECONDS + ): + continue + pending.pop(name, None) + try: + os.unlink(request_path) + except FileNotFoundError: + pass + response = response_payload( + action, + identity, + 1, + "", + failure_stderr(action, "transport-request-invalid"), + ) + else: + # Return a safe failure class, never exception text or contents. + response = response_payload( + action, + identity, + 1, + "", + failure_stderr(action, post_validation_failure_code(error)), + ) + atomic(response_path, response) + for name in names: + if not name.endswith(".ack"): + continue + identity = name[:-4] + if IDENTITY.fullmatch(identity) is None: + continue + response_path = os.path.join(session, identity + ".response") + if not os.path.exists(response_path): + continue + try: + response = json.loads(private_file(response_path).decode("utf-8", "strict")) + if copied_file(os.path.join(session, name)) != ( + identity + "\n" + ).encode("ascii"): + fail("transport-ack-invalid") + successful_release = ( + response.get("action") == "release" + and response.get("status") == 0 + ) + for suffix in (".response", ".ack"): + try: + os.unlink(os.path.join(session, identity + suffix)) + except FileNotFoundError: + pass + if successful_release: + atomic( + os.path.join(session, "released"), + (transaction + "\n").encode("ascii"), + ) + except (OSError, RuntimeError, UnicodeError, ValueError): + pass + time.sleep(0.05) + + +def main(argv: list[str] | None = None) -> int: + arguments = sys.argv[1:] if argv is None else argv + if len(arguments) != 1 or IDENTITY.fullmatch(arguments[0]) is None: + print("runtime-state-mutation-transport-broker: invalid arguments", file=sys.stderr) + return 64 + serve(arguments[0]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/lib/onboard/experimental/hermes-portable-build-context-files.ts b/src/lib/onboard/experimental/hermes-portable-build-context-files.ts index 36db676264b..a5d6208c4b3 100644 --- a/src/lib/onboard/experimental/hermes-portable-build-context-files.ts +++ b/src/lib/onboard/experimental/hermes-portable-build-context-files.ts @@ -143,6 +143,7 @@ export const HERMES_PORTABLE_BUILD_CONTEXT_FILES = [ { path: "scripts/patch-bundled-npm-tar.mts", mode: "100755" }, { path: "scripts/runtime_state_mutation_hermes_publisher.py", mode: "100755" }, { path: "scripts/runtime-state-mutation-control.py", mode: "100755" }, + { path: "scripts/runtime-state-mutation-transport-broker.py", mode: "100755" }, { path: "scripts/runtime-state-mutation-startup-gate.py", mode: "100755" }, { path: "scripts/state-dir-guard.py", mode: "100755" }, { diff --git a/src/lib/onboard/experimental/hermes-portable-build-context.ts b/src/lib/onboard/experimental/hermes-portable-build-context.ts index 589818d5dda..d1340d6e4a3 100644 --- a/src/lib/onboard/experimental/hermes-portable-build-context.ts +++ b/src/lib/onboard/experimental/hermes-portable-build-context.ts @@ -79,6 +79,7 @@ const LOCAL_COPY_SOURCES = [ "scripts/patch-bundled-npm-brace-expansion.mts", "scripts/patch-bundled-npm-tar.mts", "scripts/runtime-state-mutation-control.py", + "scripts/runtime-state-mutation-transport-broker.py", "scripts/runtime-state-mutation-startup-gate.py", "scripts/runtime_state_mutation_hermes_publisher.py", "scripts/state-dir-guard.py", diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 9e2c0c75cda..5f94a5048bd 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -26,11 +26,37 @@ import { import { createDockerOperationAuthority } from "./docker-operation-authority"; import { DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS, - createDockerStateMutationHelperTransportBrokerSource, createDockerStateMutationOwner, createDockerStateMutationSurface, } from "./docker-state-mutation"; +const TRANSPORT_BROKER = path.join( + import.meta.dirname, + "..", + "..", + "..", + "..", + "scripts", + "runtime-state-mutation-transport-broker.py", +); + +const TRANSPORT_BROKER_HARNESS = String.raw` +import importlib.util +import os +import sys + +spec = importlib.util.spec_from_file_location("runtime_state_mutation_transport_broker", sys.argv[1]) +broker = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = broker +spec.loader.exec_module(broker) +broker.ROOT = sys.argv[2] +broker.EXPECTED_UID = os.getuid() +broker.EXPECTED_GID = os.getgid() +broker.TIMEOUTS = {**broker.TIMEOUTS, "activate": float(sys.argv[3])} +broker.HELPER = sys.argv[4] +raise SystemExit(broker.main([sys.argv[5]])) +`; + function ownerThatStopsAfterPrepare(runtime: ReturnType) { const acquireMutationExecution = vi.fn(() => { throw new Error("injected controller exit before helper invocation"); @@ -103,18 +129,21 @@ async function createBrokerRuntime( const helper = path.join(root, "helper.py"); const transaction = randomBytes(32).toString("hex"); const session = path.join(root, transaction); - const uid = process.getuid?.() ?? 0; - const gid = process.getgid?.() ?? 0; - const brokerSource = createDockerStateMutationHelperTransportBrokerSource({ - root, - expectedUid: uid, - expectedGid: gid, - activateTimeoutSeconds, - }); fs.writeFileSync(helper, helperSource(root), { mode: 0o500 }); - const broker = spawn("python3", ["-I", "-c", brokerSource, helper, transaction], { - stdio: ["ignore", "ignore", "pipe"], - }); + const broker = spawn( + "python3", + [ + "-I", + "-c", + TRANSPORT_BROKER_HARNESS, + TRANSPORT_BROKER, + root, + String(activateTimeoutSeconds), + helper, + transaction, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); const brokerExit = new Promise((resolve, reject) => { broker.once("exit", () => resolve()); broker.once("error", reject); @@ -566,7 +595,7 @@ describe("Docker state mutation owner", () => { runtime.capture.mock.calls.filter( ([, args]) => args.includes("--detach") && - args.includes("/usr/local/lib/nemoclaw/runtime-state-mutation-control.py"), + args.includes("/usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py"), ), ).toHaveLength(1); expect(helperCalls.map(([, args, timeout]) => [args.at(-1), timeout])).toEqual([ diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index e413ece591a..d6e99977224 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -4,7 +4,6 @@ import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { deflateRawSync } from "node:zlib"; import type { ContainerEngine, @@ -54,6 +53,8 @@ const DOCKER_PROVIDER_ID = "docker"; const SUPPORTED_STATE_ROOT = "/sandbox/.hermes"; const HELPER_PYTHON_PATH = "/opt/hermes/.venv/bin/python3"; const HELPER_PATH = "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py"; +const HELPER_TRANSPORT_BROKER_PATH = + "/usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py"; const HELPER_FAST_TIMEOUT_MS = 30_000; const HELPER_ACTIVATION_RETRY_ACK_TIMEOUT_MS = 5_000; const HELPER_ACTIVATION_CHECKPOINT_TIMEOUT_MS = 150_000; @@ -85,8 +86,6 @@ except FileNotFoundError: except Exception: raise SystemExit(${HELPER_TRANSPORT_SESSION_INSPECTION_FAILED_STATUS}) raise SystemExit(${HELPER_TRANSPORT_SESSION_PRESENT_STATUS})`; -export const DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_BOOTSTRAP = - "import base64,sys,zlib;source=zlib.decompress(base64.b64decode(sys.argv.pop(1)),-15);exec(compile(source,'','exec'))"; const MAX_HELPER_TRANSPORT_BYTES = 128 * 1024; const MAX_INSPECTION_BYTES = 1024 * 1024; const MAX_MOUNTS = 256; @@ -103,320 +102,6 @@ const POSITIVE_DECIMAL = /^[1-9][0-9]*$/u; const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; const helperTransportPoll = new Int32Array(new SharedArrayBuffer(4)); -export interface DockerStateMutationHelperTransportBrokerSourceOptions { - readonly root: string; - readonly expectedUid: number; - readonly expectedGid: number; - readonly activateTimeoutSeconds: number; -} - -export function createDockerStateMutationHelperTransportBrokerSource( - options: DockerStateMutationHelperTransportBrokerSourceOptions, -): string { - return String.raw` -import fcntl -import hashlib -import json -import os -import re -import signal -import stat -import subprocess -import sys -import time - -ROOT = ${JSON.stringify(options.root)} -EXPECTED_UID = ${options.expectedUid} -EXPECTED_GID = ${options.expectedGid} -MAXIMUM = 128 * 1024 -TIMEOUTS = {"acquire": 30, "assert": 30, "publish": 900, "recover": 900, "rollback": 900, "activate": ${options.activateTimeoutSeconds}, "release": ${HELPER_RELEASE_TIMEOUT_MS / 1000}} -IDENTITY = re.compile(r"[a-f0-9]{64}\Z") -INCOMING = re.compile(r"([a-f0-9]{64})\.(acquire|assert|publish|recover|rollback|activate|release)\.incoming\Z") -PUBLICATION_SETTLE_SECONDS = 5 - -def fail(code): - raise RuntimeError(code) - -def directory(path): - metadata = os.lstat(path) - if (not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != EXPECTED_UID or - metadata.st_gid != EXPECTED_GID or - stat.S_IMODE(metadata.st_mode) != 0o700): - fail("transport-directory-invalid") - -def atomic(path, payload): - temporary = path + ".tmp-" + str(os.getpid()) - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, 0o600) - try: - offset = 0 - while offset < len(payload): - written = os.write(descriptor, payload[offset:]) - if written <= 0: - fail("transport-write-failed") - offset += written - os.fsync(descriptor) - finally: - os.close(descriptor) - os.replace(temporary, path) - -def private_file(path): - descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) - try: - before = os.fstat(descriptor) - payload = os.read(descriptor, MAXIMUM + 1) - after = os.fstat(descriptor) - if (not stat.S_ISREG(before.st_mode) or before.st_uid != EXPECTED_UID or - before.st_gid != EXPECTED_GID or - stat.S_IMODE(before.st_mode) != 0o600 or before.st_nlink != 1 or - len(payload) > MAXIMUM or os.read(descriptor, 1) or - (before.st_dev, before.st_ino, before.st_mode, before.st_nlink, before.st_uid, - before.st_gid, before.st_size, before.st_mtime_ns, before.st_ctime_ns) != - (after.st_dev, after.st_ino, after.st_mode, after.st_nlink, after.st_uid, - after.st_gid, after.st_size, after.st_mtime_ns, after.st_ctime_ns)): - fail("transport-file-invalid") - return payload - finally: - os.close(descriptor) - -def copied_file(path): - descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) - try: - before = os.fstat(descriptor) - payload = bytearray() - while len(payload) <= MAXIMUM: - chunk = os.read(descriptor, min(64 * 1024, MAXIMUM + 1 - len(payload))) - if not chunk: - break - payload.extend(chunk) - after = os.fstat(descriptor) - if (not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or len(payload) > MAXIMUM or - (before.st_dev, before.st_ino, before.st_nlink, before.st_uid, before.st_gid, - before.st_size, before.st_mtime_ns, before.st_ctime_ns) != - (after.st_dev, after.st_ino, after.st_nlink, after.st_uid, after.st_gid, - after.st_size, after.st_mtime_ns, after.st_ctime_ns)): - fail("transport-copied-file-invalid") - return bytes(payload) - finally: - os.close(descriptor) - -def response_payload(action, identity, status, stdout, stderr): - return json.dumps({"schemaVersion": 1, "action": action, "identity": identity, - "status": status, "stdout": stdout, "stderr": stderr}, - ensure_ascii=True, separators=(",", ":")).encode("utf-8") + b"\n" - -def failure_stderr(action, code): - return json.dumps({"schemaVersion": 1, "action": action, "status": "failed", "code": code}, - ensure_ascii=True, separators=(",", ":")) + "\n" - -def post_validation_failure_code(error): - if isinstance(error, RuntimeError): - code = str(error) - if code in ("helper-file-missing", "helper-file-invalid", "transport-response-too-large"): - return code - return "transport-runtime-failed" - if isinstance(error, UnicodeError): - return "transport-response-encoding-invalid" - if isinstance(error, FileNotFoundError): - return "transport-resource-missing" - if isinstance(error, PermissionError): - return "transport-permission-denied" - if isinstance(error, OSError): - return "transport-io-failed" - return "transport-response-invalid" - -def normalize_helper_stderr(action, status, stderr): - if not stderr: - return stderr - try: - failure = json.loads(stderr.decode("utf-8", "strict")) - if (isinstance(failure, dict) and failure.get("schemaVersion") == 1 and - failure.get("action") == action and failure.get("status") == "failed" and - isinstance(failure.get("code"), str) and - re.fullmatch(r"[a-z][a-z0-9-]{0,127}", failure["code"]) is not None): - return stderr - except (UnicodeError, ValueError): - pass - code = "helper-process-failed" if status != 0 else "helper-protocol-stderr" - return failure_stderr(action, code).encode("utf-8") - -def publisher_phase_failure(action, stderr): - if action != "publish": - return stderr - try: - failure = json.loads(stderr.decode("utf-8", "strict")) - if (not isinstance(failure, dict) or failure.get("schemaVersion") != 1 or - failure.get("action") != "publish" or failure.get("status") != "failed" or - failure.get("code") != "publisher-guard-failed"): - return stderr - journal = json.loads(private_file( - "/var/lib/nemoclaw/runtime-state-mutation/hermes-publisher.json" - ).decode("utf-8", "strict")) - operation = journal.get("operation") if isinstance(journal, dict) else None - phase = operation.get("phase") if isinstance(operation, dict) else None - if phase not in ("intent", "begun", "state-applied", "top-applied"): - return stderr - failure["code"] = "publisher-guard-" + phase + "-failed" - return (json.dumps(failure, ensure_ascii=True, separators=(",", ":")) + "\n").encode("utf-8") - except (OSError, RuntimeError, UnicodeError, ValueError): - return stderr - -def run_helper(action, request): - try: - metadata = os.lstat(helper) - except FileNotFoundError: - fail("helper-file-missing") - if (not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != EXPECTED_UID or - metadata.st_gid != EXPECTED_GID or - stat.S_IMODE(metadata.st_mode) & 0o022): - fail("helper-file-invalid") - deadline = time.monotonic() + TIMEOUTS[action] - completed = None - for _ in range(2): - process = subprocess.Popen([sys.executable, "-I", helper, action], stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) - try: - stdout, stderr = process.communicate( - request, timeout=max(0.001, deadline - time.monotonic())) - except subprocess.TimeoutExpired: - try: - os.killpg(process.pid, signal.SIGKILL) - except OSError: - try: - process.kill() - except OSError: - pass - try: - process.communicate(timeout=5) - except (subprocess.TimeoutExpired, OSError): - pass - raise - completed = subprocess.CompletedProcess( - process.args, process.returncode, stdout=stdout, stderr=stderr) - if completed.returncode >= 0: - return completed - # Replay one signal exit inside this action's deadline. - return completed - -helper = sys.argv[1] -transaction = sys.argv[2] -if IDENTITY.fullmatch(transaction) is None: - fail("transport-transaction-invalid") -os.makedirs(ROOT, mode=0o700, exist_ok=True) -directory(ROOT) -session = os.path.join(ROOT, transaction) -os.makedirs(session, mode=0o700, exist_ok=True) -directory(session) -lock = os.open(os.path.join(session, "broker.lock"), os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o600) -try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) -except BlockingIOError: - raise SystemExit(0) -atomic(os.path.join(session, "ready"), (transaction + "\n").encode("ascii")) -pending = {} - -while True: - names = sorted(os.listdir(session)) - if "released" in names and "resumed" in names: - try: - expected = (transaction + "\n").encode("ascii") - if (private_file(os.path.join(session, "released")) == expected and - copied_file(os.path.join(session, "resumed")) == expected): - for name in ("released", "resumed", "ready", "broker.lock"): - try: - os.unlink(os.path.join(session, name)) - except FileNotFoundError: - pass - try: - os.rmdir(session) - except OSError: - pass - raise SystemExit(0) - except (OSError, RuntimeError, UnicodeError, ValueError): - pass - for name in names: - incoming = INCOMING.fullmatch(name) - if incoming is None: - continue - identity, action = incoming.groups() - request_path = os.path.join(session, name) - response_path = os.path.join(session, identity + ".response") - if os.path.exists(response_path): - continue - validated = False - try: - request = copied_file(request_path) - if not request.endswith(b"\n") or hashlib.sha256(request).hexdigest() != identity: - fail("transport-request-invalid") - envelope = json.loads(request.decode("utf-8", "strict")) - if (not isinstance(envelope, dict) or envelope.get("action") != action or - envelope.get("transactionId") != transaction): - fail("transport-request-invalid") - validated = True - pending.pop(name, None) - os.unlink(request_path) - completed = run_helper(action, request) - if len(completed.stdout) > MAXIMUM or len(completed.stderr) > MAXIMUM: - fail("transport-response-too-large") - status = completed.returncode if completed.returncode >= 0 else 128 - completed.returncode - stderr = publisher_phase_failure(action, completed.stderr) - stderr = normalize_helper_stderr(action, status, stderr) - response = response_payload(action, identity, status, - completed.stdout.decode("utf-8", "strict"), stderr.decode("utf-8", "strict")) - except subprocess.TimeoutExpired: - response = response_payload(action, identity, 1, "", failure_stderr(action, "helper-timeout")) - except (OSError, RuntimeError, UnicodeError, ValueError) as error: - if not validated: - first_observed = pending.setdefault(name, time.monotonic()) - if time.monotonic() - first_observed < PUBLICATION_SETTLE_SECONDS: - continue - pending.pop(name, None) - try: - os.unlink(request_path) - except FileNotFoundError: - pass - response = response_payload(action, identity, 1, "", - failure_stderr(action, "transport-request-invalid")) - else: - # Preserve a safe, actionable failure class without returning - # exception text, host paths, or request contents to the caller. - response = response_payload(action, identity, 1, "", - failure_stderr(action, post_validation_failure_code(error))) - atomic(response_path, response) - for name in names: - if not name.endswith(".ack"): - continue - identity = name[:-4] - if IDENTITY.fullmatch(identity) is None: - continue - response_path = os.path.join(session, identity + ".response") - if not os.path.exists(response_path): - continue - try: - response = json.loads(private_file(response_path).decode("utf-8", "strict")) - if copied_file(os.path.join(session, name)) != (identity + "\n").encode("ascii"): - fail("transport-ack-invalid") - successful_release = response.get("action") == "release" and response.get("status") == 0 - for suffix in (".response", ".ack"): - try: - os.unlink(os.path.join(session, identity + suffix)) - except FileNotFoundError: - pass - if successful_release: - atomic(os.path.join(session, "released"), (transaction + "\n").encode("ascii")) - except (OSError, RuntimeError, UnicodeError, ValueError): - pass - time.sleep(0.05) -`; -} - -export const DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE = - createDockerStateMutationHelperTransportBrokerSource({ - root: HELPER_TRANSPORT_ROOT, - expectedUid: 0, - expectedGid: 0, - activateTimeoutSeconds: DOCKER_STATE_MUTATION_ACTIVATE_TIMEOUT_MS / 1000, - }); - type HelperAction = | "acquire" | "assert" @@ -1388,12 +1073,7 @@ function helperTransportBrokerCommand( runtimeId, HELPER_PYTHON_PATH, "-I", - "-c", - DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_BOOTSTRAP, - deflateRawSync(Buffer.from(DOCKER_STATE_MUTATION_HELPER_TRANSPORT_BROKER_SOURCE, "utf8"), { - level: 9, - }).toString("base64"), - HELPER_PATH, + HELPER_TRANSPORT_BROKER_PATH, transactionId, ]), targetIndex: 5, diff --git a/test/agents/hermes/hermes-doctor-config-hash.test.ts b/test/agents/hermes/hermes-doctor-config-hash.test.ts index e2a159f3d94..2c5e3742a81 100644 --- a/test/agents/hermes/hermes-doctor-config-hash.test.ts +++ b/test/agents/hermes/hermes-doctor-config-hash.test.ts @@ -121,6 +121,10 @@ describe("Hermes doctor and config hash boundary", () => { ); const stateLockPlanPath = path.join(tmp, "state-lock-plan.json"); const runtimeStateMutationControlPath = path.join(libDir, "runtime-state-mutation-control.py"); + const runtimeStateMutationTransportBrokerPath = path.join( + libDir, + "runtime-state-mutation-transport-broker.py", + ); const runtimeStateMutationStartupGatePath = path.join( libDir, "runtime-state-mutation-startup-gate.py", @@ -168,6 +172,7 @@ describe("Hermes doctor and config hash boundary", () => { mcpCredentialBoundaryPath, path.join(libDir, "state-dir-guard.py"), runtimeStateMutationControlPath, + runtimeStateMutationTransportBrokerPath, runtimeStateMutationStartupGatePath, runtimeStateMutationPublisherPath, stateLockPlanPath, @@ -184,6 +189,7 @@ describe("Hermes doctor and config hash boundary", () => { } fs.writeFileSync(runtimeStateMutationControlPath, "# controller fixture\n"); + fs.writeFileSync(runtimeStateMutationTransportBrokerPath, "# broker fixture\n"); fs.writeFileSync(runtimeStateMutationStartupGatePath, "# startup gate fixture\n"); fs.writeFileSync(runtimeStateMutationPublisherPath, "# publisher fixture\n"); fs.writeFileSync(path.join(libDir, "hermes-runtime-config-guard.py"), "# guard fixture\n"); @@ -219,7 +225,7 @@ describe("Hermes doctor and config hash boundary", () => { expect(result.stderr).toBe(""); expect(fs.readFileSync(chownLogPath, "utf-8")).toBe( [ - `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${runtimeStateMutationControlPath} ${runtimeStateMutationStartupGatePath} ${runtimeStateMutationPublisherPath} ${stateLockPlanPath} ${runtimeStateMutationCapabilityPath} ${path.join(libDir, "managed-gateway-control.py")} ${buildMcpDigestPath} ${hermesCronRestoreControlPath} ${mcpCredentialBoundaryPath}`, + `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${runtimeStateMutationControlPath} ${runtimeStateMutationTransportBrokerPath} ${runtimeStateMutationStartupGatePath} ${runtimeStateMutationPublisherPath} ${stateLockPlanPath} ${runtimeStateMutationCapabilityPath} ${path.join(libDir, "managed-gateway-control.py")} ${buildMcpDigestPath} ${hermesCronRestoreControlPath} ${mcpCredentialBoundaryPath}`, `-R 0:0 ${preloadsDir}`, "", ].join("\n"), @@ -237,6 +243,7 @@ describe("Hermes doctor and config hash boundary", () => { expect(mode(path.join(libDir, "gateway-supervisor.sh"))).toBe("444"); expect(mode(path.join(libDir, "state-dir-guard.py"))).toBe("500"); expect(mode(runtimeStateMutationControlPath)).toBe("500"); + expect(mode(runtimeStateMutationTransportBrokerPath)).toBe("500"); expect(mode(runtimeStateMutationStartupGatePath)).toBe("555"); expect(mode(runtimeStateMutationPublisherPath)).toBe("500"); expect(mode(stateLockPlanPath)).toBe("444"); diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index a5f57cb1266..9dfc4f9841a 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -50,6 +50,7 @@ real_resume_reference = control._resume_reference real_prove_released_activation = control._prove_released_activation real_prove_parent_acknowledged_activation = control._prove_parent_acknowledged_activation real_resume_acknowledged_parent = control._resume_acknowledged_parent +real_transport_broker_reference = control._transport_broker_reference control._assert_private_procfs = lambda: None control._open_activation_guard_pidfd = lambda _reference: os.open( os.devnull, os.O_RDONLY @@ -186,6 +187,44 @@ def process(pid, state, parent, start, uid, command, inode, executable_inode=Non executable_inode, ) +fixed_transport_broker = process( + 88, + "S", + 1, + "788", + control.ROOT_UID, + ( + b"/opt/hermes/.venv/bin/python3", + b"-I", + control.TRANSPORT_BROKER_PATH, + b"a" * 64, + ), + 188, +) +dynamic_transport_broker = process( + 89, + "S", + 1, + "789", + control.ROOT_UID, + ( + b"/opt/hermes/.venv/bin/python3", + b"-I", + b"-c", + b"dynamic-source", + b"encoded-source", + b"/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", + b"a" * 64, + ), + 189, +) +control._capture_process = lambda _pid: fixed_transport_broker +fixed_transport_broker_reference = real_transport_broker_reference() +results = {"fixed_transport_broker": fixed_transport_broker_reference.pid} +control._capture_process = lambda _pid: dynamic_transport_broker +results["dynamic_transport_broker_rejected"] = real_transport_broker_reference() is None +control._capture_process = real_capture_process + root_uid = control.ROOT_UID pid1 = process(1, "S", 0, "100", root_uid, (control.OPENSHELL_ARGV0,), 101) stopped_pid1 = process(1, "T", 0, "100", root_uid, (control.OPENSHELL_ARGV0,), 101) @@ -250,7 +289,6 @@ activation = control.ActivationProof( ), ) -results = {} with tempfile.TemporaryDirectory() as process_probe: process_metadata = os.stat(process_probe, follow_symlinks=False) real_os_stat = control.os.stat diff --git a/test/runtime/sandbox/sandbox-provisioning.test.ts b/test/runtime/sandbox/sandbox-provisioning.test.ts index 864fe771bef..38bed4d5fdd 100644 --- a/test/runtime/sandbox/sandbox-provisioning.test.ts +++ b/test/runtime/sandbox/sandbox-provisioning.test.ts @@ -1052,6 +1052,10 @@ describe("Hermes sandbox provisioning", () => { localLib, "runtime-state-mutation-control.py", ); + const runtimeStateMutationTransportBrokerPath = path.join( + localLib, + "runtime-state-mutation-transport-broker.py", + ); const runtimeStateMutationStartupGatePath = path.join( localLib, "runtime-state-mutation-startup-gate.py", @@ -1103,6 +1107,7 @@ describe("Hermes sandbox provisioning", () => { gatewaySupervisorPath, stateDirGuardPath, runtimeStateMutationControlPath, + runtimeStateMutationTransportBrokerPath, runtimeStateMutationStartupGatePath, runtimeStateMutationPublisherPath, stateLockPlanPath, @@ -1141,7 +1146,7 @@ describe("Hermes sandbox provisioning", () => { expect(result.status, result.stderr).toBe(0); expect(calls).toContain( - `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${runtimeStateMutationControlPath} ${runtimeStateMutationStartupGatePath} ${runtimeStateMutationPublisherPath} ${stateLockPlanPath} ${runtimeStateMutationCapabilityPath} ${managedGatewayControlPath} ${buildMcpDigestPath} ${hermesCronRestoreControlPath} ${mcpManifest}`, + `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${runtimeStateMutationControlPath} ${runtimeStateMutationTransportBrokerPath} ${runtimeStateMutationStartupGatePath} ${runtimeStateMutationPublisherPath} ${stateLockPlanPath} ${runtimeStateMutationCapabilityPath} ${managedGatewayControlPath} ${buildMcpDigestPath} ${hermesCronRestoreControlPath} ${mcpManifest}`, ); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(hermesCronRestoreControlPath).mode & 0o777).toString(8)).toBe("700"); @@ -1154,6 +1159,9 @@ describe("Hermes sandbox provisioning", () => { expect((fs.statSync(corporateCaRuntimePath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(runtimeStateMutationControlPath).mode & 0o777).toString(8)).toBe("500"); + expect( + (fs.statSync(runtimeStateMutationTransportBrokerPath).mode & 0o777).toString(8), + ).toBe("500"); expect((fs.statSync(runtimeStateMutationStartupGatePath).mode & 0o777).toString(8)).toBe( "555", ); diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 77853d62ba9..82cea076b87 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -74,6 +74,8 @@ describe("runtime state mutation controller", () => { start_identity_drift: "start-process-identity-drift", startup_support_identity_drift: "startup-support-identity-drift", running_supervisor_hold: "supervisor-not-host-stopped", + fixed_transport_broker: 88, + dynamic_transport_broker_rejected: true, }); expect(harnessResult.hold_events).toEqual([ ["stop", 10], From e2b8c814b5ab61dcab396277921ecc38815e1aee Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 16:23:21 -0700 Subject: [PATCH 41/44] refactor(runtime): close broker review findings --- docs/reference/troubleshooting.mdx | 4 +- ...runtime-state-mutation-transport-broker.py | 33 +--- .../docker-state-mutation.test.ts | 8 +- .../hermes/hermes-doctor-config-hash.test.ts | 161 ------------------ .../sandbox/sandbox-provisioning.test.ts | 147 ---------------- 5 files changed, 10 insertions(+), 343 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 422ba625ffe..094a76a98ac 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1834,10 +1834,10 @@ Do not kill or manually restart the held entrypoint. Run `$$nemoclaw shields status` from the host so NemoClaw can recover the retained target and authenticate the startup release. If recovery still fails, preserve the owner-only lifecycle ledger and the complete error for diagnosis instead of removing a marker or changing in-sandbox permissions. -If the startup gate instead reports `invalid-state code= transaction=` and Hermes refuses startup, the fixed code identifies rejected gate state rather than an active transition that can make progress by waiting. +If the startup gate instead reports `invalid-state code= transaction=` and Hermes refuses startup, the error code identifies rejected gate state rather than an active transition that can make progress by waiting. The transaction is a 64-character identifier when the gate safely parsed it, or `unknown` when it could not. Run `$$nemoclaw shields status` from the host to recover the retained transition. -If status cannot recover it, preserve `~/.nemoclaw/state/runtime-provider-lifecycle/`, the fixed code, and the transaction identifier for diagnosis; do not remove or edit the root-owned in-sandbox receipts. +If status cannot recover it, preserve `~/.nemoclaw/state/runtime-provider-lifecycle/`, the error code, and the transaction identifier for diagnosis; do not remove or edit the root-owned in-sandbox receipts. ### Hermes startup reports `HERMES_CONFIG_MUTATION_ORPHANED` diff --git a/scripts/runtime-state-mutation-transport-broker.py b/scripts/runtime-state-mutation-transport-broker.py index b8dda952187..b56552aa567 100755 --- a/scripts/runtime-state-mutation-transport-broker.py +++ b/scripts/runtime-state-mutation-transport-broker.py @@ -250,36 +250,6 @@ def normalize_helper_stderr(action: str, status_code: int, stderr: bytes) -> byt return failure_stderr(action, code).encode("utf-8") -def publisher_phase_failure(action: str, stderr: bytes) -> bytes: - if action != "publish": - return stderr - try: - failure = json.loads(stderr.decode("utf-8", "strict")) - if ( - not isinstance(failure, dict) - or failure.get("schemaVersion") != 1 - or failure.get("action") != "publish" - or failure.get("status") != "failed" - or failure.get("code") != "publisher-guard-failed" - ): - return stderr - journal = json.loads( - private_file( - "/var/lib/nemoclaw/runtime-state-mutation/hermes-publisher.json" - ).decode("utf-8", "strict") - ) - operation = journal.get("operation") if isinstance(journal, dict) else None - phase = operation.get("phase") if isinstance(operation, dict) else None - if phase not in ("intent", "begun", "state-applied", "top-applied"): - return stderr - failure["code"] = "publisher-guard-" + phase + "-failed" - return ( - json.dumps(failure, ensure_ascii=True, separators=(",", ":")) + "\n" - ).encode("utf-8") - except (OSError, RuntimeError, UnicodeError, ValueError): - return stderr - - def run_helper(action: str, request: bytes) -> subprocess.CompletedProcess[bytes]: try: metadata = os.lstat(HELPER) @@ -406,8 +376,7 @@ def serve(transaction: str) -> None: if completed.returncode >= 0 else 128 - completed.returncode ) - stderr = publisher_phase_failure(action, completed.stderr) - stderr = normalize_helper_stderr(action, status_code, stderr) + stderr = normalize_helper_stderr(action, status_code, completed.stderr) response = response_payload( action, identity, diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts index 5f94a5048bd..87453cc0f18 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.test.ts @@ -217,6 +217,9 @@ if action == "acquire": if action == "assert": os.write(2, b"unexpected helper stderr") if action == "publish": + os.write(2, b'{"schemaVersion":1,"action":"publish","status":"failed","code":"publisher-guard-failed"}\\n') + raise SystemExit(1) +if action == "rollback": os.write(1, b"\\xff") `, ); @@ -228,7 +231,10 @@ if action == "publish": const protocol = await sendBrokerRequest(runtime, "assert"); expect(brokerFailureCode(protocol.response)).toBe("helper-protocol-stderr"); - const encoding = await sendBrokerRequest(runtime, "publish"); + const publisher = await sendBrokerRequest(runtime, "publish"); + expect(brokerFailureCode(publisher.response)).toBe("publisher-guard-failed"); + + const encoding = await sendBrokerRequest(runtime, "rollback"); expect(brokerFailureCode(encoding.response)).toBe("transport-response-encoding-invalid"); fs.unlinkSync(runtime.helper); diff --git a/test/agents/hermes/hermes-doctor-config-hash.test.ts b/test/agents/hermes/hermes-doctor-config-hash.test.ts index 2c5e3742a81..63ccab50539 100644 --- a/test/agents/hermes/hermes-doctor-config-hash.test.ts +++ b/test/agents/hermes/hermes-doctor-config-hash.test.ts @@ -97,167 +97,6 @@ describe("Hermes doctor and config hash boundary", () => { } }); - it("locks trusted gateway recovery preloads as image-owned read-only files", () => { - const dockerfile = fs.readFileSync(HERMES_DOCKERFILE, "utf-8"); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-preload-lock-")); - const binDir = path.join(tmp, "usr-local-bin"); - const libDir = path.join(tmp, "usr-local-lib-nemoclaw"); - const preloadsDir = path.join(libDir, "preloads"); - const buildMcpDigestPath = path.join(libDir, "build-hermes-mcp-digest.py"); - const mcpConfigTransactionPath = path.join(libDir, "hermes-mcp-config-transaction.py"); - const langfuseCredentialPatcherPath = path.join( - libDir, - "patch-hermes-langfuse-credentials.mts", - ); - const discordRecoveryPatcherPath = path.join( - libDir, - "patch-hermes-discord-recovery-permissions.py", - ); - const profilePolicyPatcherPath = path.join(libDir, "patch-hermes-profile-policy-defaults.py"); - const managedPolicyReaderPath = path.join(libDir, "managed_policy.py"); - const mcpCredentialBoundaryPath = path.join( - libDir, - "openshell-child-visible-credentials.v0.0.106.json", - ); - const stateLockPlanPath = path.join(tmp, "state-lock-plan.json"); - const runtimeStateMutationControlPath = path.join(libDir, "runtime-state-mutation-control.py"); - const runtimeStateMutationTransportBrokerPath = path.join( - libDir, - "runtime-state-mutation-transport-broker.py", - ); - const runtimeStateMutationStartupGatePath = path.join( - libDir, - "runtime-state-mutation-startup-gate.py", - ); - const runtimeStateMutationPublisherPath = path.join( - libDir, - "runtime_state_mutation_hermes_publisher.py", - ); - const runtimeStateMutationCapabilityPath = path.join( - tmp, - "runtime-state-mutation-publisher-v1.json", - ); - const hermesCronRestoreControlPath = path.join(libDir, "hermes-cron-restore-control.py"); - const nestedDir = path.join(preloadsDir, "nested"); - const profileDir = path.join(tmp, "etc-profile.d"); - const bashrcPath = path.join(tmp, "bash.bashrc"); - const chownLogPath = path.join(tmp, "chown.log"); - const mode = (entry: string) => (fs.statSync(entry).mode & 0o777).toString(8); - - try { - fs.mkdirSync(binDir, { recursive: true }); - fs.mkdirSync(nestedDir, { recursive: true, mode: 0o777 }); - fs.mkdirSync(profileDir, { recursive: true }); - for (const fixturePath of [ - path.join(binDir, "nemoclaw-start"), - path.join(binDir, "nemoclaw-managed-startup-hold"), - path.join(binDir, "nemoclaw-managed-bootstrap"), - path.join(binDir, "nemoclaw-gateway-control"), - path.join(libDir, "corporate-ca-runtime.sh"), - path.join(libDir, "entrypoint-env-wrapper.sh"), - path.join(libDir, "sandbox-init.sh"), - path.join(libDir, "gateway-supervisor.sh"), - path.join(libDir, "validate-hermes-env-secret-boundary.py"), - path.join(libDir, "patch-hermes-session-list-preview.py"), - path.join(libDir, "patch-hermes-sqlite-temp-store.py"), - discordRecoveryPatcherPath, - profilePolicyPatcherPath, - managedPolicyReaderPath, - langfuseCredentialPatcherPath, - path.join(libDir, "seed-hermes-dashboard-config.py"), - path.join(libDir, "hermes-runtime-config-guard.py"), - path.join(libDir, "finalize-tirith-marker.py"), - buildMcpDigestPath, - mcpConfigTransactionPath, - mcpCredentialBoundaryPath, - path.join(libDir, "state-dir-guard.py"), - runtimeStateMutationControlPath, - runtimeStateMutationTransportBrokerPath, - runtimeStateMutationStartupGatePath, - runtimeStateMutationPublisherPath, - stateLockPlanPath, - runtimeStateMutationCapabilityPath, - path.join(libDir, "managed-gateway-control.py"), - hermesCronRestoreControlPath, - path.join(libDir, "sandbox-rlimits.sh"), - path.join(preloadsDir, "gateway-safety-net.js"), - path.join(nestedDir, "ciao-preload.js"), - bashrcPath, - ]) { - fs.mkdirSync(path.dirname(fixturePath), { recursive: true }); - fs.writeFileSync(fixturePath, "test\n", { mode: 0o666 }); - } - - fs.writeFileSync(runtimeStateMutationControlPath, "# controller fixture\n"); - fs.writeFileSync(runtimeStateMutationTransportBrokerPath, "# broker fixture\n"); - fs.writeFileSync(runtimeStateMutationStartupGatePath, "# startup gate fixture\n"); - fs.writeFileSync(runtimeStateMutationPublisherPath, "# publisher fixture\n"); - fs.writeFileSync(path.join(libDir, "hermes-runtime-config-guard.py"), "# guard fixture\n"); - - const lockCommand = dockerRunCommandBetween( - dockerfile, - "# Dockerfile.base is the source of truth for rlimit hooks.", - "# Flatten stale published base images", - ) - .replaceAll("/usr/local/bin", binDir) - .replaceAll("/usr/local/lib/nemoclaw", libDir) - .replaceAll("/opt/hermes/.venv/bin/python3", "python3") - .replaceAll("/usr/local/share/nemoclaw/state-lock-plan.json", stateLockPlanPath) - .replaceAll( - "/usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json", - runtimeStateMutationCapabilityPath, - ) - .replaceAll("/etc/profile.d", profileDir) - .replaceAll("/etc/bash.bashrc", bashrcPath); - const script = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `chown_log=${JSON.stringify(chownLogPath)}`, - 'chown() { printf "%s\\n" "$*" >> "$chown_log"; }', - lockCommand, - ].join("\n"); - const result = spawnSync("bash", ["-c", script], { - encoding: "utf-8", - timeout: 5000, - }); - - expect(result.status, result.stderr).toBe(0); - expect(result.stderr).toBe(""); - expect(fs.readFileSync(chownLogPath, "utf-8")).toBe( - [ - `root:root ${path.join(binDir, "nemoclaw-gateway-control")} ${path.join(libDir, "gateway-supervisor.sh")} ${path.join(libDir, "state-dir-guard.py")} ${runtimeStateMutationControlPath} ${runtimeStateMutationTransportBrokerPath} ${runtimeStateMutationStartupGatePath} ${runtimeStateMutationPublisherPath} ${stateLockPlanPath} ${runtimeStateMutationCapabilityPath} ${path.join(libDir, "managed-gateway-control.py")} ${buildMcpDigestPath} ${hermesCronRestoreControlPath} ${mcpCredentialBoundaryPath}`, - `-R 0:0 ${preloadsDir}`, - "", - ].join("\n"), - ); - expect(mode(path.join(binDir, "nemoclaw-gateway-control"))).toBe("700"); - expect(mode(hermesCronRestoreControlPath)).toBe("700"); - expect(mode(path.join(libDir, "finalize-tirith-marker.py"))).toBe("755"); - expect(mode(mcpConfigTransactionPath)).toBe("755"); - expect(mode(discordRecoveryPatcherPath)).toBe("755"); - expect(mode(profilePolicyPatcherPath)).toBe("755"); - expect(mode(managedPolicyReaderPath)).toBe("444"); - expect(mode(langfuseCredentialPatcherPath)).toBe("444"); - expect(mode(mcpCredentialBoundaryPath)).toBe("444"); - expect(mode(buildMcpDigestPath)).toBe("444"); - expect(mode(path.join(libDir, "gateway-supervisor.sh"))).toBe("444"); - expect(mode(path.join(libDir, "state-dir-guard.py"))).toBe("500"); - expect(mode(runtimeStateMutationControlPath)).toBe("500"); - expect(mode(runtimeStateMutationTransportBrokerPath)).toBe("500"); - expect(mode(runtimeStateMutationStartupGatePath)).toBe("555"); - expect(mode(runtimeStateMutationPublisherPath)).toBe("500"); - expect(mode(stateLockPlanPath)).toBe("444"); - expect(mode(runtimeStateMutationCapabilityPath)).toBe("444"); - expect(mode(path.join(libDir, "managed-gateway-control.py"))).toBe("500"); - expect(mode(preloadsDir)).toBe("755"); - expect(mode(nestedDir)).toBe("755"); - expect(mode(path.join(preloadsDir, "gateway-safety-net.js"))).toBe("444"); - expect(mode(path.join(nestedDir, "ciao-preload.js"))).toBe("444"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - it("keeps upstream doctor changes out of generated config hash inputs", () => { const dockerfile = fs.readFileSync(HERMES_DOCKERFILE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-doctor-lock-")); diff --git a/test/runtime/sandbox/sandbox-provisioning.test.ts b/test/runtime/sandbox/sandbox-provisioning.test.ts index 38bed4d5fdd..53a173e258e 100644 --- a/test/runtime/sandbox/sandbox-provisioning.test.ts +++ b/test/runtime/sandbox/sandbox-provisioning.test.ts @@ -1029,153 +1029,6 @@ describe("sandbox provisioning: base runtime tools", () => { }); describe("Hermes sandbox provisioning", () => { - it("stages privileged lifecycle helpers with root-only Hermes image modes", () => { - const dockerfile = fs.readFileSync(HERMES_DOCKERFILE, "utf-8"); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-helper-modes-")); - const localBin = path.join(tmp, "usr", "local", "bin"); - const localLib = path.join(tmp, "usr", "local", "lib", "nemoclaw"); - const etcDir = path.join(tmp, "etc"); - const profileDir = path.join(etcDir, "profile.d"); - const bashrcPath = path.join(etcDir, "bash.bashrc"); - const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); - const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); - const buildMcpDigestPath = path.join(localLib, "build-hermes-mcp-digest.py"); - const mcpConfigTransactionPath = path.join(localLib, "hermes-mcp-config-transaction.py"); - const langfuseCredentialPatcherPath = path.join( - localLib, - "patch-hermes-langfuse-credentials.mts", - ); - const managedPolicyReaderPath = path.join(localLib, "managed_policy.py"); - const mcpManifest = path.join(localLib, "openshell-child-visible-credentials.v0.0.106.json"); - const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); - const runtimeStateMutationControlPath = path.join( - localLib, - "runtime-state-mutation-control.py", - ); - const runtimeStateMutationTransportBrokerPath = path.join( - localLib, - "runtime-state-mutation-transport-broker.py", - ); - const runtimeStateMutationStartupGatePath = path.join( - localLib, - "runtime-state-mutation-startup-gate.py", - ); - const runtimeStateMutationPublisherPath = path.join( - localLib, - "runtime_state_mutation_hermes_publisher.py", - ); - const stateLockPlanPath = path.join( - tmp, - "usr", - "local", - "share", - "nemoclaw", - "state-lock-plan.json", - ); - const runtimeStateMutationCapabilityPath = path.join( - tmp, - "usr", - "local", - "share", - "nemoclaw", - "runtime-state-mutation-publisher-v1.json", - ); - const managedGatewayControlPath = path.join(localLib, "managed-gateway-control.py"); - const hermesCronRestoreControlPath = path.join(localLib, "hermes-cron-restore-control.py"); - const corporateCaRuntimePath = path.join(localLib, "corporate-ca-runtime.sh"); - const files = [ - path.join(localBin, "nemoclaw-start"), - path.join(localBin, "nemoclaw-managed-startup-hold"), - path.join(localBin, "nemoclaw-managed-bootstrap"), - gatewayControlPath, - corporateCaRuntimePath, - path.join(localLib, "entrypoint-env-wrapper.sh"), - path.join(localLib, "sandbox-init.sh"), - path.join(localLib, "validate-hermes-env-secret-boundary.py"), - path.join(localLib, "patch-hermes-session-list-preview.py"), - path.join(localLib, "patch-hermes-sqlite-temp-store.py"), - path.join(localLib, "patch-hermes-discord-recovery-permissions.py"), - path.join(localLib, "patch-hermes-profile-policy-defaults.py"), - managedPolicyReaderPath, - langfuseCredentialPatcherPath, - path.join(localLib, "seed-hermes-dashboard-config.py"), - path.join(localLib, "hermes-runtime-config-guard.py"), - path.join(localLib, "finalize-tirith-marker.py"), - buildMcpDigestPath, - mcpConfigTransactionPath, - mcpManifest, - gatewaySupervisorPath, - stateDirGuardPath, - runtimeStateMutationControlPath, - runtimeStateMutationTransportBrokerPath, - runtimeStateMutationStartupGatePath, - runtimeStateMutationPublisherPath, - stateLockPlanPath, - runtimeStateMutationCapabilityPath, - managedGatewayControlPath, - hermesCronRestoreControlPath, - path.join(localLib, "sandbox-rlimits.sh"), - ]; - const command = dockerRunCommandBetween( - dockerfile, - "# Dockerfile.base is the source of truth for rlimit hooks.", - "# Wrap the hermes CLI", - ) - .replaceAll("/usr/local/bin", localBin) - .replaceAll("/usr/local/lib/nemoclaw", localLib) - .replaceAll("/opt/hermes/.venv/bin/python3", "python3") - .replaceAll("/usr/local/share/nemoclaw/state-lock-plan.json", stateLockPlanPath) - .replaceAll( - "/usr/local/share/nemoclaw/runtime-state-mutation-publisher-v1.json", - runtimeStateMutationCapabilityPath, - ) - .replaceAll("/etc/profile.d", profileDir) - .replaceAll("/etc/bash.bashrc", bashrcPath); - try { - fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(localLib, { recursive: true }); - fs.mkdirSync(path.dirname(stateLockPlanPath), { recursive: true }); - fs.mkdirSync(etcDir, { recursive: true }); - fs.writeFileSync(bashrcPath, "# fixture\n", { mode: 0o600 }); - files.forEach((file) => { - fs.writeFileSync(file, "# fixture\n", { mode: 0o600 }); - }); - const { result, calls } = runLoggedDockerShell(command, tmp, [ - 'chown() { printf "chown %s\\n" "$*" >> "$call_log"; }', - ]); - - expect(result.status, result.stderr).toBe(0); - expect(calls).toContain( - `chown root:root ${gatewayControlPath} ${gatewaySupervisorPath} ${stateDirGuardPath} ${runtimeStateMutationControlPath} ${runtimeStateMutationTransportBrokerPath} ${runtimeStateMutationStartupGatePath} ${runtimeStateMutationPublisherPath} ${stateLockPlanPath} ${runtimeStateMutationCapabilityPath} ${managedGatewayControlPath} ${buildMcpDigestPath} ${hermesCronRestoreControlPath} ${mcpManifest}`, - ); - expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); - expect((fs.statSync(hermesCronRestoreControlPath).mode & 0o777).toString(8)).toBe("700"); - expect((fs.statSync(mcpConfigTransactionPath).mode & 0o777).toString(8)).toBe("755"); - expect((fs.statSync(langfuseCredentialPatcherPath).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(mcpManifest).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(buildMcpDigestPath).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(managedPolicyReaderPath).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(corporateCaRuntimePath).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); - expect((fs.statSync(runtimeStateMutationControlPath).mode & 0o777).toString(8)).toBe("500"); - expect( - (fs.statSync(runtimeStateMutationTransportBrokerPath).mode & 0o777).toString(8), - ).toBe("500"); - expect((fs.statSync(runtimeStateMutationStartupGatePath).mode & 0o777).toString(8)).toBe( - "555", - ); - expect((fs.statSync(runtimeStateMutationPublisherPath).mode & 0o777).toString(8)).toBe("500"); - expect((fs.statSync(stateLockPlanPath).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(runtimeStateMutationCapabilityPath).mode & 0o777).toString(8)).toBe( - "444", - ); - expect((fs.statSync(managedGatewayControlPath).mode & 0o777).toString(8)).toBe("500"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - function runHermesPathValidation(pathEntriesBeforeManifest: string[] = []) { const dockerfile = fs.readFileSync(HERMES_DOCKERFILE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-path-")); From a2a59e2d7bf5de277231232f41518d0ad076626a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 16:57:12 -0700 Subject: [PATCH 42/44] refactor(runtime): close final advisor findings --- .../runtime-provider/docker-state-mutation.ts | 27 +++++++------------ .../sandbox/sandbox-rlimit-hooks.test.ts | 10 +++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index d6e99977224..7067a2e20c4 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -1164,14 +1164,6 @@ function writePrivateTransportFile(filePath: string, value: Buffer): void { } } -function copyHelperTransportFile( - capture: HelperTransportCapture, - command: PersistedEngineLifecycleExactCommand, - timeoutMs = HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, -): ContainerEngineCommandResult { - return capture(command, timeoutMs); -} - function readHelperTransportFile( capture: HelperTransportCapture, runtimeId: string, @@ -1187,8 +1179,7 @@ function readHelperTransportFile( const remainingMs = deadline - Date.now(); if (remainingMs <= 0) fail(`root helper ${action} transport response did not arrive`); fs.rmSync(destination, { force: true }); - const result = copyHelperTransportFile( - capture, + const result = capture( helperTransportCopyFromCommand(runtimeId, containerPath, destination), Math.min(HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, remainingMs), ); @@ -1220,13 +1211,13 @@ function probeHelperTransport( ): boolean { return withHelperTransportHostDirectory(options.hostTransportRoot, (temporary) => { const destination = path.join(temporary, "ready"); - const result = copyHelperTransportFile( - capture, + const result = capture( helperTransportCopyFromCommand( options.runtimeId, `${helperTransportSessionPath(transactionId)}/ready`, destination, ), + HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, ); if (result.error || result.status !== 0 || result.stderr.length !== 0) return false; const ready = fs.readFileSync(destination); @@ -1302,13 +1293,13 @@ function finishReleasedHelperTransport( const resumed = path.join(temporary, "resumed"); writePrivateTransportFile(resumed, Buffer.from(`${transactionId}\n`, "ascii")); requireCommandSuccess( - copyHelperTransportFile( - capture, + capture( helperTransportCopyToCommand( options.runtimeId, resumed, `${helperTransportSessionPath(transactionId)}/resumed`, ), + HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, ), "root helper transport release finalization", ); @@ -1398,13 +1389,13 @@ function invokeHelperTransport( const request = path.join(temporary, "request"); writePrivateTransportFile(request, input); requireCommandSuccess( - copyHelperTransportFile( - capture, + capture( helperTransportCopyToCommand( options.runtimeId, request, `${sessionPath}/${identity}.${action}.incoming`, ), + HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, ), "root helper transport request publication", ); @@ -1420,13 +1411,13 @@ function invokeHelperTransport( const acknowledgement = path.join(temporary, "ack"); writePrivateTransportFile(acknowledgement, Buffer.from(`${identity}\n`, "ascii")); requireCommandSuccess( - copyHelperTransportFile( - capture, + capture( helperTransportCopyToCommand( options.runtimeId, acknowledgement, `${sessionPath}/${identity}.ack`, ), + HELPER_TRANSPORT_COMMAND_TIMEOUT_MS, ), "root helper transport response acknowledgement", ); diff --git a/test/runtime/sandbox/sandbox-rlimit-hooks.test.ts b/test/runtime/sandbox/sandbox-rlimit-hooks.test.ts index 6ddfd06848e..9fa70a9ba79 100644 --- a/test/runtime/sandbox/sandbox-rlimit-hooks.test.ts +++ b/test/runtime/sandbox/sandbox-rlimit-hooks.test.ts @@ -622,6 +622,10 @@ describe("sandbox rlimit system hooks (#2173)", () => { const gatewaySupervisor = path.join(localLib, "gateway-supervisor.sh"); const stateDirGuard = path.join(localLib, "state-dir-guard.py"); const runtimeStateMutationControl = path.join(localLib, "runtime-state-mutation-control.py"); + const runtimeStateMutationTransportBroker = path.join( + localLib, + "runtime-state-mutation-transport-broker.py", + ); const runtimeStateMutationStartupGate = path.join( localLib, "runtime-state-mutation-startup-gate.py", @@ -673,6 +677,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { fs.writeFileSync(gatewaySupervisor, "# gateway supervisor fixture\n"); fs.writeFileSync(stateDirGuard, "# state-dir guard fixture\n"); fs.writeFileSync(runtimeStateMutationControl, "# runtime mutation control fixture\n"); + fs.writeFileSync(runtimeStateMutationTransportBroker, "# runtime mutation broker fixture\n"); fs.writeFileSync(runtimeStateMutationStartupGate, "# runtime mutation gate fixture\n"); fs.writeFileSync(runtimeStateMutationPublisher, "# runtime mutation publisher fixture\n"); fs.writeFileSync(stateLockPlan, "{}\n"); @@ -739,6 +744,10 @@ describe("sandbox rlimit system hooks (#2173)", () => { "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", runtimeStateMutationControl, ) + .replaceAll( + "/usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py", + runtimeStateMutationTransportBroker, + ) .replaceAll( "/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py", runtimeStateMutationStartupGate, @@ -785,6 +794,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { expect(fs.statSync(mcpCredentialBoundary).mode & 0o777).toBe(0o444); expect(fs.statSync(buildMcpDigest).mode & 0o777).toBe(0o444); expect(fs.statSync(runtimeStateMutationControl).mode & 0o777).toBe(0o500); + expect(fs.statSync(runtimeStateMutationTransportBroker).mode & 0o777).toBe(0o500); expect(fs.statSync(runtimeStateMutationStartupGate).mode & 0o777).toBe(0o555); expect(fs.statSync(runtimeStateMutationPublisher).mode & 0o777).toBe(0o500); expect(fs.statSync(stateLockPlan).mode & 0o777).toBe(0o444); From 4bfd3f6621843b8a2db201f565674c14109c15e9 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 17:41:58 -0700 Subject: [PATCH 43/44] fix(runtime): bind broker interpreter identity Signed-off-by: Prekshi Vyas --- scripts/runtime-state-mutation-control.py | 19 ++++++- .../runtime-state-mutation-control-harness.ts | 54 ++++++++++++++++++- .../runtime-state-mutation-control.test.ts | 2 + 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/scripts/runtime-state-mutation-control.py b/scripts/runtime-state-mutation-control.py index 595c6d677cd..a8bc278202a 100755 --- a/scripts/runtime-state-mutation-control.py +++ b/scripts/runtime-state-mutation-control.py @@ -117,6 +117,7 @@ START_LOG_DRAIN_PATHS = (b"tee", b"/usr/bin/tee", b"/bin/tee") STARTUP_GATE_PYTHON = b"/opt/hermes/.venv/bin/python3" STARTUP_GATE_HELPER = b"/usr/local/lib/nemoclaw/runtime-state-mutation-startup-gate.py" +TRANSPORT_BROKER_PYTHON = b"/opt/hermes/.venv/bin/python3" TRANSPORT_BROKER_PATH = ( b"/usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py" ) @@ -3180,18 +3181,34 @@ def _resume_activation_guard_pidfd(pidfd: int) -> None: def _transport_broker_reference() -> ProcessReference | None: + try: + executable_before = os.stat(TRANSPORT_BROKER_PYTHON) + except OSError: + return None process = _capture_process(os.getppid()) if process is None: return None + try: + executable_after = os.stat(TRANSPORT_BROKER_PYTHON) + except OSError: + return None command = process.command if ( - process.pid <= 1 + not stat.S_ISREG(executable_before.st_mode) + or executable_before.st_uid != ROOT_UID + or executable_before.st_gid != ROOT_GID + or stat.S_IMODE(executable_before.st_mode) & 0o022 + or not _same_filesystem_object(executable_before, executable_after) + or process.pid <= 1 or process.state in ("Z", "X", "x") or process.uids != (ROOT_UID,) * 4 or len(command) != 4 + or command[0] != TRANSPORT_BROKER_PYTHON or command[1] != b"-I" or command[2] != TRANSPORT_BROKER_PATH or re.fullmatch(rb"[0-9a-f]{64}", command[3]) is None + or process.executable_key() + != (executable_after.st_dev, executable_after.st_ino) ): return None return _process_reference(process) diff --git a/test/helpers/runtime-state-mutation-control-harness.ts b/test/helpers/runtime-state-mutation-control-harness.ts index 9dfc4f9841a..783d06c4dab 100644 --- a/test/helpers/runtime-state-mutation-control-harness.ts +++ b/test/helpers/runtime-state-mutation-control-harness.ts @@ -201,12 +201,42 @@ fixed_transport_broker = process( ), 188, ) -dynamic_transport_broker = process( +forged_transport_broker = process( 89, "S", 1, "789", control.ROOT_UID, + ( + control.TRANSPORT_BROKER_PYTHON, + b"-I", + control.TRANSPORT_BROKER_PATH, + b"a" * 64, + ), + 189, + fixed_transport_broker.executable_inode + 1, +) +wrong_argv_transport_broker = process( + 90, + "S", + 1, + "790", + control.ROOT_UID, + ( + b"/tmp/forged-python", + b"-I", + control.TRANSPORT_BROKER_PATH, + b"a" * 64, + ), + 190, + fixed_transport_broker.executable_inode, +) +dynamic_transport_broker = process( + 91, + "S", + 1, + "791", + control.ROOT_UID, ( b"/opt/hermes/.venv/bin/python3", b"-I", @@ -216,14 +246,34 @@ dynamic_transport_broker = process( b"/usr/local/lib/nemoclaw/runtime-state-mutation-control.py", b"a" * 64, ), - 189, + 191, +) +real_os_stat = control.os.stat +transport_broker_executable = types.SimpleNamespace( + st_mode=stat.S_IFREG | 0o755, + st_uid=control.ROOT_UID, + st_gid=control.ROOT_GID, + st_dev=fixed_transport_broker.executable_device, + st_ino=fixed_transport_broker.executable_inode, +) +control.os.stat = lambda path, *args, **kwargs: ( + transport_broker_executable + if path == control.TRANSPORT_BROKER_PYTHON + else real_os_stat(path, *args, **kwargs) ) control._capture_process = lambda _pid: fixed_transport_broker fixed_transport_broker_reference = real_transport_broker_reference() results = {"fixed_transport_broker": fixed_transport_broker_reference.pid} +control._capture_process = lambda _pid: forged_transport_broker +results["forged_transport_broker_rejected"] = real_transport_broker_reference() is None +control._capture_process = lambda _pid: wrong_argv_transport_broker +results["wrong_argv_transport_broker_rejected"] = ( + real_transport_broker_reference() is None +) control._capture_process = lambda _pid: dynamic_transport_broker results["dynamic_transport_broker_rejected"] = real_transport_broker_reference() is None control._capture_process = real_capture_process +control.os.stat = real_os_stat root_uid = control.ROOT_UID pid1 = process(1, "S", 0, "100", root_uid, (control.OPENSHELL_ARGV0,), 101) diff --git a/test/state/runtime-state-mutation-control.test.ts b/test/state/runtime-state-mutation-control.test.ts index 82cea076b87..5d91d9722c2 100644 --- a/test/state/runtime-state-mutation-control.test.ts +++ b/test/state/runtime-state-mutation-control.test.ts @@ -75,6 +75,8 @@ describe("runtime state mutation controller", () => { startup_support_identity_drift: "startup-support-identity-drift", running_supervisor_hold: "supervisor-not-host-stopped", fixed_transport_broker: 88, + forged_transport_broker_rejected: true, + wrong_argv_transport_broker_rejected: true, dynamic_transport_broker_rejected: true, }); expect(harnessResult.hold_events).toEqual([ From 8f10786334a98e4e6f2307d90149682299739770 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Sat, 29 Aug 2026 17:49:46 -0700 Subject: [PATCH 44/44] docs(shields): clarify immutable lock fallback Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 44001583900..2a57988e930 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -5560,7 +5560,7 @@ The following flags change defaults for commands that manage existing sandboxes. | `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH` | `1` to enable | Skips the automatic trusted container recreation during `$$nemoclaw recover` when two managed scans find no supervisor while PID 1 remains stable. Use only as a troubleshooting escape hatch; recovery then falls back to the rebuild or re-onboard guidance. | | `NEMOCLAW_SHIELDS_ACCEPT_LEGACY_BASELINE` | `1` to opt in | Allows advanced immutable-config verification to trust the current on-disk bytes for older or partial content baselines. Use only after you have rebuilt or manually inspected the sandbox state and accepted that the baseline is operator-approved. | -| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive whole-number milliseconds (default `750`, maximum `10000`) | NemoClaw waits this long after re-applying a config lockdown before checking that the lock still holds. It applies during ordinary `$$nemoclaw shields up` transitions, shields auto-restore, and `shields up` drift remediation. Values above `10000` use `10000`. Fractional, zero, negative, blank, and invalid values use the default. If NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This check narrows the window in which an in-sandbox reconciler can revert permissions; it does not eliminate that window. The best-effort `chattr +i` immutable bit remains the only fully durable lock. Raise the value on hosts where the gateway settles slowly. | +| `NEMOCLAW_SHIELDS_SETTLE_MS` | positive whole-number milliseconds (default `750`, maximum `10000`) | NemoClaw waits this long after re-applying a config lockdown before checking that the lock still holds. It applies during ordinary `$$nemoclaw shields up` transitions, shields auto-restore, and `shields up` drift remediation. Values above `10000` use `10000`. Fractional, zero, negative, blank, and invalid values use the default. If NemoClaw cannot re-confirm the lock within the retry budget, shields stay down. This check narrows the window in which an in-sandbox reconciler can revert permissions; it does not eliminate that window. When the best-effort `chattr +i` operation succeeds, its immutable bit provides the durable lock. If that operation is unavailable or fails, no durable lock is available. Raise the value on hosts where the gateway settles slowly. | | `NEMOCLAW_SKIP_UNREACHABLE_SANDBOX_BACKUP` | Exactly `1` to opt in (`true`, `yes`, `0` are not accepted) | Applies to standalone `$$nemoclaw backup-all` runs. Skips running sandboxes whose in-sandbox SSH endpoint does not answer. It does not relax the installer's strict pre-upgrade backup, which still aborts if any registered sandbox is skipped or fails. Any uncommitted state since the last successful backup is not included in the skipped backup. | | `NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS` | `1` to opt in | Makes `$$nemoclaw uninstall` remove every gateway port on the host instead of only the port `NEMOCLAW_GATEWAY_PORT` selects. Equivalent to passing the `--all-gateway-ports` flag; the whole-host `Proceed?` confirmation still applies unless `--yes` is also passed. Each port runs as its own uninstall, and the variable is dropped from those runs so the sweep cannot re-enter itself. | | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA` | `1` to opt in | Acknowledges data loss during `$$nemoclaw uninstall`, skips eligible fresh sandbox backups, and removes the otherwise-preserved entries (`rebuild-backups/`, `backups/`, `sandboxes.json`) in the selected gateway's state root. It does not select the explicit `--destroy-user-data` CLI-shim removal path; shim handling follows the ordinary selected-gateway scope. The global `Proceed?` confirmation still applies unless `--yes` is also passed. |