diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 1810c3bf66c..12b21a80025 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -271,7 +271,24 @@ export function managedImageOpenShellBasePolicyPath(agent: ManagedStartupAgent): ); } -function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout = 20_000) { +export interface ManagedImageCommandResult { + readonly error?: Error; + readonly status: number | null; + readonly stdout: string | null; + readonly stderr: string | null; +} + +export type ManagedImageCommandRunner = ( + argv: readonly string[], + env: NodeJS.ProcessEnv, + timeout?: number, +) => ManagedImageCommandResult; + +function commandResult( + argv: readonly string[], + env: NodeJS.ProcessEnv, + timeout = 20_000, +): ManagedImageCommandResult { const [command, ...args] = argv; if (!command) throw new Error("command argv must not be empty"); return spawnSync(command, args, { @@ -283,13 +300,13 @@ function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout }); } -function commandDetail(result: ReturnType): string { +function commandDetail(result: ManagedImageCommandResult): string { return `${result.error?.message ?? ""}\n${result.stdout ?? ""}\n${result.stderr ?? ""}` .trim() .slice(-8_000); } -function isDockerNotFound(result: ReturnType): boolean { +function isDockerNotFound(result: ManagedImageCommandResult): boolean { return ( result.status !== 0 && /(?:no such (?:container|network|object)|not found)/iu.test(commandDetail(result)) @@ -390,25 +407,65 @@ export function managedImageOpenShellProbe( : agent === "hermes" ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null" : "/usr/local/bin/dcode --version >/dev/null"; + const readinessLabel = + agent === "openclaw" + ? "OpenClaw health endpoint" + : agent === "hermes" + ? "Hermes health endpoint" + : "LangChain Deep Agents Code version command"; + const probeStep = (label: string, command: string) => + `if ! ${command}; then\n printf '%s\\n' ${JSON.stringify( + `managed-image startup probe failed: ${label}`, + )} >&2\n exit 1\nfi`; return [ - "set -eu", - `test -x ${ - agent === "openclaw" - ? "/usr/local/bin/openclaw" - : agent === "hermes" - ? "/usr/local/bin/hermes" - : "/usr/local/bin/dcode" - }`, - `grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`, - "test ! -L /run/nemoclaw/managed-startup-runtime.env", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', - "test ! -L /run/nemoclaw/managed-startup-complete.json", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', - "test -s /usr/local/share/nemoclaw/corporate-ca.pem", - 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', - "test -s /run/nemoclaw/managed-startup-ca-bundle.pem", - 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', - healthProbe, + "set -u", + probeStep( + `${agent} executable`, + `test -x ${ + agent === "openclaw" + ? "/usr/local/bin/openclaw" + : agent === "hermes" + ? "/usr/local/bin/hermes" + : "/usr/local/bin/dcode" + }`, + ), + probeStep( + `${agent} managed model configuration`, + `grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`, + ), + probeStep( + "managed runtime environment must not be a symbolic link", + "test ! -L /run/nemoclaw/managed-startup-runtime.env", + ), + probeStep( + "managed runtime environment owner, group, and mode must equal 0:0:444", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', + ), + probeStep( + "managed startup completion must not be a symbolic link", + "test ! -L /run/nemoclaw/managed-startup-complete.json", + ), + probeStep( + "managed startup completion owner, group, and mode must equal 0:0:444", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', + ), + probeStep( + "corporate CA file must exist and be nonempty", + "test -s /usr/local/share/nemoclaw/corporate-ca.pem", + ), + probeStep( + "corporate CA owner, group, and mode must equal 0:0:444", + 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', + ), + probeStep( + "managed startup CA bundle must exist and be nonempty", + "test -s /run/nemoclaw/managed-startup-ca-bundle.pem", + ), + probeStep( + "managed startup CA bundle owner, group, and mode must equal 0:0:444", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', + ), + probeStep(readinessLabel, healthProbe), ].join("\n"); } @@ -631,8 +688,12 @@ function parseImmutableManifestReference(image: string): { }; } -function resolveLocalImageContentId(image: string, env: NodeJS.ProcessEnv): string { - const inspect = commandResult(["docker", "image", "inspect", "--format", "{{.Id}}", image], env); +function resolveLocalImageContentId( + image: string, + env: NodeJS.ProcessEnv, + runCommand: ManagedImageCommandRunner = commandResult, +): string { + const inspect = runCommand(["docker", "image", "inspect", "--format", "{{.Id}}", image], env); const contentId = String(inspect.stdout ?? "").trim(); if (inspect.status !== 0 || !/^sha256:[a-f0-9]{64}$/u.test(contentId)) { throw new Error( @@ -642,25 +703,31 @@ function resolveLocalImageContentId(image: string, env: NodeJS.ProcessEnv): stri return contentId; } +export function managedImageHarnessContainerListArgs( + sandbox: string, + includeStopped: boolean, +): string[] { + return [ + "docker", + "ps", + includeStopped ? "-aq" : "-q", + "--no-trunc", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${sandbox}`, + ]; +} + function exactHarnessContainerIds( input: Inputs, networkName: string, env: NodeJS.ProcessEnv, + includeStopped = true, + runCommand: ManagedImageCommandRunner = commandResult, ): { candidateCount: number; exactIds: string[] } { - const expectedContentId = resolveLocalImageContentId(input.image, env); - const list = commandResult( - [ - "docker", - "ps", - "-aq", - "--no-trunc", - "--filter", - "label=openshell.ai/managed-by=openshell", - "--filter", - `label=openshell.ai/sandbox-name=${input.sandbox}`, - ], - env, - ); + const expectedContentId = resolveLocalImageContentId(input.image, env, runCommand); + const list = runCommand(managedImageHarnessContainerListArgs(input.sandbox, includeStopped), env); if (list.status !== 0) { throw new Error(`could not resolve the OpenShell sandbox container: ${commandDetail(list)}`); } @@ -670,7 +737,7 @@ function exactHarnessContainerIds( .filter(Boolean); const exactIds: string[] = []; for (const candidate of candidates) { - const inspect = commandResult(["docker", "inspect", candidate], env); + const inspect = runCommand(["docker", "inspect", candidate], env); if (inspect.status !== 0) continue; try { const records = JSON.parse(String(inspect.stdout ?? "")) as Array<{ @@ -694,20 +761,38 @@ function exactHarnessContainerIds( return { candidateCount: candidates.length, exactIds }; } -function assertExactSandboxImage( +export function assertExactSandboxImage( input: Inputs, networkName: string, env: NodeJS.ProcessEnv, + runCommand: ManagedImageCommandRunner = commandResult, ): string { - const resolved = exactHarnessContainerIds(input, networkName, env); + // A managed-bootstrap transaction keeps its stopped rollback backup until + // commit. Qualify the one running replacement here; rollback cleanup below + // continues to inspect every stopped and running container. + const resolved = exactHarnessContainerIds(input, networkName, env, false, runCommand); if (resolved.candidateCount !== 1 || resolved.exactIds.length !== 1) { throw new Error( - `OpenShell did not launch exactly one harness-owned PR image container: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact`, + `OpenShell did not launch exactly one running harness-owned PR image container: found ${resolved.candidateCount} running labeled and ${resolved.exactIds.length} running exact`, ); } return resolved.exactIds[0] ?? ""; } +export function assertFailedBootstrapContainerCleanup( + input: Inputs, + networkName: string, + env: NodeJS.ProcessEnv, + runCommand: ManagedImageCommandRunner = commandResult, +): void { + const resolved = exactHarnessContainerIds(input, networkName, env, true, runCommand); + if (resolved.candidateCount !== 0 || resolved.exactIds.length !== 0) { + throw new Error( + `managed-bootstrap rollback retained a failed held sandbox: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact containers`, + ); + } +} + function assertFailedSandboxAbsent( onboard: OnboardModule, input: Inputs, @@ -928,12 +1013,7 @@ async function run([ + ["docker image inspect", { status: 0, stdout: `${contentId}\n`, stderr: "" }], + [`docker ps ${listScope}`, { status: 0, stdout: listOutput, stderr: "" }], + [`docker inspect ${containerId}`, managedContainerInspectResult(contentId)], + ]); + return (argv) => { + calls.push([...argv]); + return responses.get(argv.slice(0, 3).join(" ")) ?? SUCCESS_WITHOUT_OUTPUT; + }; +} + describe("protected managed-image runtime contract", () => { it("binds the public and protected managed-image plans to one supervisor argv (#7744)", () => { const authorityStore = {}; @@ -127,6 +175,65 @@ describe("protected managed-image runtime contract", () => { expect(fs.existsSync(stateDir)).toBe(false); }); + it("qualifies the running exact image before rollback cleanup (#7744)", () => { + const calls: string[][] = []; + const contentId = `sha256:${"b".repeat(64)}`; + const containerId = "c".repeat(64); + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const runningCommand = createManagedImageCommandRunner( + contentId, + containerId, + "-q", + `${containerId}\n`, + calls, + ); + const cleanedCommand = createManagedImageCommandRunner( + contentId, + containerId, + "-aq", + "", + calls, + ); + + expect(assertExactSandboxImage(input, "managed-network", {}, runningCommand)).toBe(containerId); + assertFailedBootstrapContainerCleanup(input, "managed-network", {}, cleanedCommand); + + expect(calls.filter((argv) => argv[1] === "ps").map((argv) => argv[2])).toEqual(["-q", "-aq"]); + }); + + it("rejects a stopped labeled container after failed bootstrap cleanup (#7744)", () => { + const contentId = `sha256:${"b".repeat(64)}`; + const containerId = "c".repeat(64); + const input = parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + VALID_SANDBOX, + ]); + const runCommand = createManagedImageCommandRunner( + contentId, + containerId, + "-aq", + `${containerId}\n`, + [], + ); + + expect(() => + assertFailedBootstrapContainerCleanup(input, "managed-network", {}, runCommand), + ).toThrow( + "managed-bootstrap rollback retained a failed held sandbox: found 1 labeled and 1 exact containers", + ); + }); + it("assigns every protected agent and route a unique OpenShell-compatible sandbox name (#8497)", () => { const routeKinds = [...MANAGED_IMAGE_LOCAL_INFERENCE_KINDS, "rollback"] as const; const qualifications = PROTECTED_MANAGED_IMAGE_AGENTS.flatMap((agent) => @@ -254,7 +361,22 @@ describe("protected managed-image runtime contract", () => { sandbox: managedImageProtectedSandboxName(agent, "nim"), }); expect(path.isAbsolute(managedImageOpenShellBasePolicyPath(agent))).toBe(true); - expect(managedImageOpenShellProbe(agent)).toContain("managed-startup-complete.json"); + const probe = managedImageOpenShellProbe(agent); + const syntax = spawnSync("/bin/sh", ["-n", "-c", probe], { encoding: "utf8" }); + expect(syntax.status, syntax.stderr).toBe(0); + expect(probe).toContain("managed-startup-complete.json"); + expect(probe).toContain( + `managed-image startup probe failed: ${ + agent === "openclaw" + ? "OpenClaw health endpoint" + : agent === "hermes" + ? "Hermes health endpoint" + : "LangChain Deep Agents Code version command" + }`, + ); + expect(probe).toContain( + "managed-image startup probe failed: managed startup completion owner, group, and mode must equal 0:0:444", + ); }); it("rewrites only the inference route while preserving the managed agent profile", () => {