From e4013f265a6b96391b05e943b19dab3cd4fbadcc Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 17 Aug 2026 10:39:05 -0700 Subject: [PATCH 1/2] fix(portable): reconcile timed-out stop state Signed-off-by: Senthil Ravichandran --- .../portable-demo-lifecycle-stop.test.ts | 315 ++++++++++++++++++ .../experimental/portable-demo-lifecycle.ts | 40 ++- 2 files changed, 348 insertions(+), 7 deletions(-) create mode 100644 src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts diff --git a/src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts b/src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts new file mode 100644 index 00000000000..fa0393e4b42 --- /dev/null +++ b/src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts @@ -0,0 +1,315 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { PodmanSocketAuthorityDeps } from "../../adapters/podman"; +import type { CheckpointPortableRuntimeAuthority } from "../../state/onboard-checkpoint-types"; +import { + installPortableDemoSandboxLifecycle, + type PortableDemoLifecycleDeps, + portableDemoLifecycleInternals, + type PortablePodmanLifecycleCommandResult, + stopPortableDemoSandboxLifecycle, +} from "./portable-demo-lifecycle"; + +const CONTAINER_ID = "a".repeat(64); +const SANDBOX_ID = "sandbox-id-alpha"; +const SOCKET_PATH = "/run/user/1001/podman/podman.sock"; +const RUNTIME_AUTHORITY: CheckpointPortableRuntimeAuthority = { + schemaVersion: 1, + kind: "podman", + ownership: "current-user", + uid: 1001, + homeDir: "/home/tester", + configHome: "/home/tester/.config", + runtimeDir: "/run/user/1001", + socketPath: SOCKET_PATH, +}; +const STARTUP_ARGV = [ + "env", + "CHAT_UI_URL=http://127.0.0.1:18789", + "NEMOCLAW_DASHBOARD_PORT=18789", + "OPENCLAW_HOME=/sandbox", + "OPENCLAW_STATE_DIR=/sandbox/.openclaw", + "OPENCLAW_WORKSPACE_DIR=/sandbox/.openclaw/workspace", + "NEMOCLAW_SANDBOX_NAME=alpha", + "/usr/local/bin/nemoclaw-start", +]; + +const temporaryDirectories: string[] = []; + +function socketAuthorityDeps(): PodmanSocketAuthorityDeps { + const directoryInodes = new Map(); + return { + uid: 1001, + lstat: (filePath) => { + const socket = filePath === SOCKET_PATH; + const directoryInode = directoryInodes.get(filePath) ?? BigInt(7000 + directoryInodes.size); + directoryInodes.set(filePath, directoryInode); + return { + dev: 8n, + ino: socket ? 9001n : directoryInode, + mode: socket ? 0o660n : filePath === path.dirname(SOCKET_PATH) ? 0o700n : 0o755n, + uid: socket ? 1001n : filePath.startsWith("/run/user/1001") ? 1001n : 0n, + isDirectory: () => !socket, + isSocket: () => socket, + }; + }, + }; +} + +function createPodman() { + let running = true; + let containerId = CONTAINER_ID; + const podman = vi.fn( + (args: readonly string[], _env?: NodeJS.ProcessEnv): PortablePodmanLifecycleCommandResult => { + const command = args[0] === "--url" ? args.slice(2) : args; + switch (command[0]) { + case "version": + return { status: 0, stdout: JSON.stringify({ Server: { Version: "5.6.1" } }) }; + case "ps": + return { status: 0, stdout: `${CONTAINER_ID}\n` }; + case "inspect": + return { + status: 0, + stdout: JSON.stringify([ + { + Id: containerId, + Name: `openshell-default--alpha-${SANDBOX_ID}`, + Config: { + Labels: { + "openshell.managed": "true", + "openshell.ai/sandbox-id": SANDBOX_ID, + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-namespace": "", + "openshell.ai/sandbox-workspace": "default", + }, + }, + State: { Running: running }, + }, + ]), + }; + case "stop": + running = false; + return { status: 0 }; + case "update": + return { status: 0 }; + default: + throw new Error(`Unexpected Podman command: ${args.join(" ")}`); + } + }, + ); + return { + podman, + setContainerId(value: string) { + containerId = value; + }, + setRunning(value: boolean) { + running = value; + }, + }; +} + +function lifecycleDeps( + stateDir: string, + podman: ReturnType["podman"], + overrides: Partial = {}, +): PortableDemoLifecycleDeps { + return { + platform: "linux", + podman, + podmanSocketAuthorityDeps: socketAuthorityDeps(), + stateDir, + hardenSocketDirectory: vi.fn(), + runtimeReadiness: { + uid: 1001, + home: RUNTIME_AUTHORITY.homeDir, + systemctl: () => ({ status: 0 }), + podmanCapture: () => ({ + status: 0, + stdout: JSON.stringify({ Server: { Version: "5.6.1" } }), + stderr: "", + }), + }, + log: vi.fn(), + ...overrides, + }; +} + +function createStopHarness() { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-stop-")); + temporaryDirectories.push(stateDir); + const runtime = createPodman(); + installPortableDemoSandboxLifecycle( + "alpha", + STARTUP_ARGV, + { HOME: stateDir, NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, + { + ...lifecycleDeps(stateDir, runtime.podman), + runtimeAuthority: RUNTIME_AUTHORITY, + }, + ); + const receiptPath = portableDemoLifecycleInternals.receiptPath("alpha", stateDir); + const receiptBefore = fs.readFileSync(receiptPath, "utf8"); + const originalPodman = runtime.podman.getMockImplementation()!; + runtime.podman.mockClear(); + return { originalPodman, receiptBefore, receiptPath, runtime, stateDir }; +} + +function stopSandbox( + harness: ReturnType, + overrides: Partial = {}, + beforeStop = vi.fn(), +) { + return stopPortableDemoSandboxLifecycle( + "alpha", + { + agent: "openclaw", + gatewayName: "nemoclaw", + lifecycleGeneration: CONTAINER_ID, + openshellDriver: "docker", + }, + beforeStop, + lifecycleDeps(harness.stateDir, harness.runtime.podman, overrides), + ); +} + +function timedOutStop( + harness: ReturnType, + afterStop?: (command: readonly string[]) => PortablePodmanLifecycleCommandResult | undefined, +) { + const timeout = Object.assign(new Error("spawnSync podman ETIMEDOUT"), { + code: "ETIMEDOUT", + }); + let stopAttempted = false; + harness.runtime.podman.mockImplementation((args, env) => { + const command = args[0] === "--url" ? args.slice(2) : args; + if (command[0] === "stop") { + stopAttempted = true; + return { status: null, error: timeout }; + } + if (stopAttempted) { + const result = afterStop?.(command); + if (result) return result; + } + return harness.originalPodman(args, env); + }); +} + +function expectOnlyExactStopAndInspects(harness: ReturnType): void { + const commands = harness.runtime.podman.mock.calls.map(([args]) => + args[0] === "--url" ? args.slice(2) : args, + ); + expect(commands.filter(([command]) => command === "stop")).toEqual([["stop", CONTAINER_ID]]); + expect(commands.every(([command]) => command === "inspect" || command === "stop")).toBe(true); +} + +function expectReceiptUnchanged(harness: ReturnType): void { + expect(fs.readFileSync(harness.receiptPath, "utf8")).toBe(harness.receiptBefore); +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("portable demo sandbox stop reconciliation", () => { + it("reconciles an ETIMEDOUT stop to the exact receipt-owned container state (#9200)", () => { + const harness = createStopHarness(); + let inspectionsAfterStop = 0; + let now = 0; + timedOutStop(harness, (command) => { + if (command[0] === "inspect") { + inspectionsAfterStop += 1; + if (inspectionsAfterStop === 2) harness.runtime.setRunning(false); + } + return undefined; + }); + const beforeStop = vi.fn(); + + expect( + stopSandbox( + harness, + { + now: () => now, + sleep: (milliseconds) => { + now += milliseconds; + }, + }, + beforeStop, + ), + ).toEqual({ kind: "stopped" }); + + expect(beforeStop).toHaveBeenCalledExactlyOnceWith(); + expect(now).toBe(1_000); + expectReceiptUnchanged(harness); + expectOnlyExactStopAndInspects(harness); + }); + + it("fails after bounded reconciliation when an ETIMEDOUT container remains running (#9200)", () => { + const harness = createStopHarness(); + let now = 0; + timedOutStop(harness); + + expect(() => + stopSandbox(harness, { + now: () => now, + sleep: (milliseconds) => { + now += milliseconds; + }, + }), + ).toThrow("ETIMEDOUT"); + + expect(now).toBe(30_000); + expectReceiptUnchanged(harness); + expectOnlyExactStopAndInspects(harness); + }); + + it("rejects container identity drift while reconciling an ETIMEDOUT stop (#9200)", () => { + const harness = createStopHarness(); + timedOutStop(harness, (command) => { + if (command[0] === "inspect") harness.runtime.setContainerId("b".repeat(64)); + return undefined; + }); + + expect(() => stopSandbox(harness, { now: () => 0, sleep: vi.fn() })).toThrow( + "OpenShell identity does not match sandbox 'alpha'", + ); + + expectReceiptUnchanged(harness); + expectOnlyExactStopAndInspects(harness); + }); + + it("rejects a missing receipt-owned container while reconciling an ETIMEDOUT stop (#9200)", () => { + const harness = createStopHarness(); + timedOutStop(harness, (command) => + command[0] === "inspect" ? { status: 125, stderr: "Error: no such container" } : undefined, + ); + + expect(() => stopSandbox(harness, { now: () => 0, sleep: vi.fn() })).toThrow( + "no longer has its recorded Podman container", + ); + + expectReceiptUnchanged(harness); + expectOnlyExactStopAndInspects(harness); + }); + + it("rejects an inspection failure while reconciling an ETIMEDOUT stop (#9200)", () => { + const harness = createStopHarness(); + timedOutStop(harness, (command) => + command[0] === "inspect" ? { status: 125, stderr: "permission denied" } : undefined, + ); + + expect(() => stopSandbox(harness, { now: () => 0, sleep: vi.fn() })).toThrow( + "Inspecting portable sandbox 'alpha' failed: exit 125", + ); + + expectReceiptUnchanged(harness); + expectOnlyExactStopAndInspects(harness); + }); +}); diff --git a/src/lib/onboard/experimental/portable-demo-lifecycle.ts b/src/lib/onboard/experimental/portable-demo-lifecycle.ts index dd889cb63ce..98931101bcf 100644 --- a/src/lib/onboard/experimental/portable-demo-lifecycle.ts +++ b/src/lib/onboard/experimental/portable-demo-lifecycle.ts @@ -53,6 +53,7 @@ const MAX_RECEIPT_DIRECTORY_ENTRIES = 1024; const COMMAND_TIMEOUT_MS = 30_000; const PROBE_TIMEOUT_MS = 5_000; const EXEC_READY_TIMEOUT_MS = 90_000; +const STOP_RECONCILIATION_TIMEOUT_MS = 30_000; const STARTUP_STOP_TIMEOUT_MS = 30_000; const STARTUP_TIMEOUT_MS = 90_000; const OLLAMA_STARTUP_TIMEOUT_MS = 30_000; @@ -258,6 +259,10 @@ function requireCommand(result: CommandResult, action: string): void { throw new Error(`${action} failed: ${commandDetail(result)}`); } +function isCommandTimeout(result: CommandResult): boolean { + return (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT"; +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -1312,13 +1317,34 @@ export function stopPortableDemoSandboxLifecycle( if (!inspection.running) return { kind: "already-stopped" }; beforeStop(); - requireCommand( - authority.podman(["stop", receipt.containerId]), - `Stopping portable sandbox '${sandboxName}'`, - ); - const stopped = inspectPodmanContainer(receipt.containerId, sandboxName, authority.podman); - requireReceiptOwnedInspection(receipt, stopped); - if (stopped.running) { + const stop = authority.podman(["stop", receipt.containerId]); + const inspectStoppedState = (): boolean => { + const result = authority.podman(["inspect", receipt.containerId]); + if (isMissingPodmanContainer(result)) { + throw new Error( + `Portable sandbox '${sandboxName}' no longer has its recorded Podman container`, + ); + } + const stopped = inspectPodmanContainer( + receipt.containerId, + sandboxName, + authority.podman, + result, + ); + requireReceiptOwnedInspection(receipt, stopped); + return !stopped.running; + }; + if (isCommandTimeout(stop)) { + // The rootless Podman service can continue an accepted stop after its CLI + // client times out. Reconcile only the exact receipt-owned container; do + // not retry the mutation or weaken socket and container identity checks. + const timing = { now: deps.now ?? Date.now, sleep: deps.sleep ?? defaultSleep }; + if (waitFor(STOP_RECONCILIATION_TIMEOUT_MS, timing, inspectStoppedState)) { + return { kind: "stopped" }; + } + } + requireCommand(stop, `Stopping portable sandbox '${sandboxName}'`); + if (!inspectStoppedState()) { throw new Error(`Portable sandbox '${sandboxName}' did not enter the stopped state`); } return { kind: "stopped" }; From dbc7d765ad68959988f1e30ccef4111d17c63bcf Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 17 Aug 2026 14:03:46 -0700 Subject: [PATCH 2/2] test(portable): keep stop scenarios linear Signed-off-by: Senthil Ravichandran --- .../portable-demo-lifecycle-stop.test.ts | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts b/src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts index fa0393e4b42..ed71b1e8620 100644 --- a/src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts +++ b/src/lib/onboard/experimental/portable-demo-lifecycle-stop.test.ts @@ -188,18 +188,41 @@ function timedOutStop( let stopAttempted = false; harness.runtime.podman.mockImplementation((args, env) => { const command = args[0] === "--url" ? args.slice(2) : args; - if (command[0] === "stop") { - stopAttempted = true; - return { status: null, error: timeout }; - } - if (stopAttempted) { - const result = afterStop?.(command); - if (result) return result; + switch (command[0]) { + case "stop": + stopAttempted = true; + return { status: null, error: timeout }; + default: { + const result = stopAttempted ? afterStop?.(command) : undefined; + return result ?? harness.originalPodman(args, env); + } } - return harness.originalPodman(args, env); }); } +function stopAfterSecondInspection(harness: ReturnType) { + let inspectionsAfterStop = 0; + return (command: readonly string[]): undefined => { + switch (command[0]) { + case "inspect": + inspectionsAfterStop += 1; + switch (inspectionsAfterStop) { + case 2: + harness.runtime.setRunning(false); + } + } + }; +} + +function replaceContainerOnInspection(harness: ReturnType) { + return (command: readonly string[]): undefined => { + switch (command[0]) { + case "inspect": + harness.runtime.setContainerId("b".repeat(64)); + } + }; +} + function expectOnlyExactStopAndInspects(harness: ReturnType): void { const commands = harness.runtime.podman.mock.calls.map(([args]) => args[0] === "--url" ? args.slice(2) : args, @@ -221,15 +244,8 @@ afterEach(() => { describe("portable demo sandbox stop reconciliation", () => { it("reconciles an ETIMEDOUT stop to the exact receipt-owned container state (#9200)", () => { const harness = createStopHarness(); - let inspectionsAfterStop = 0; let now = 0; - timedOutStop(harness, (command) => { - if (command[0] === "inspect") { - inspectionsAfterStop += 1; - if (inspectionsAfterStop === 2) harness.runtime.setRunning(false); - } - return undefined; - }); + timedOutStop(harness, stopAfterSecondInspection(harness)); const beforeStop = vi.fn(); expect( @@ -272,10 +288,7 @@ describe("portable demo sandbox stop reconciliation", () => { it("rejects container identity drift while reconciling an ETIMEDOUT stop (#9200)", () => { const harness = createStopHarness(); - timedOutStop(harness, (command) => { - if (command[0] === "inspect") harness.runtime.setContainerId("b".repeat(64)); - return undefined; - }); + timedOutStop(harness, replaceContainerOnInspection(harness)); expect(() => stopSandbox(harness, { now: () => 0, sleep: vi.fn() })).toThrow( "OpenShell identity does not match sandbox 'alpha'",