diff --git a/src/lib/agent-onboard.ts b/src/lib/agent-onboard.ts index e632ebfdea4..efe5151a1d7 100644 --- a/src/lib/agent-onboard.ts +++ b/src/lib/agent-onboard.ts @@ -10,6 +10,7 @@ import os from "os"; import path from "path"; import { ROOT, run } from "./runner"; +import { dockerBuild, dockerImageInspect } from "./docker"; import { loadAgent, resolveAgentName, type AgentDefinition } from "./agent-defs"; import { getProviderSelectionConfig } from "./inference-config"; import * as onboardSession from "./onboard-session"; @@ -61,13 +62,13 @@ export function createAgentSandbox(agent: AgentDefinition): { if (baseDockerfile) { const baseImageTag = `ghcr.io/nvidia/nemoclaw/${agent.name}-sandbox-base:latest`; - const inspectResult = run(["docker", "image", "inspect", baseImageTag], { + const inspectResult = dockerImageInspect(baseImageTag, { ignoreError: true, suppressOutput: true, }); if (inspectResult.status !== 0) { console.log(` Building ${agent.displayName} base image (first time only)...`); - run(["docker", "build", "-f", baseDockerfile, "-t", baseImageTag, ROOT], { + dockerBuild(baseDockerfile, baseImageTag, ROOT, { stdio: ["ignore", "inherit", "inherit"], }); console.log(` \u2713 Base image built: ${baseImageTag}`); diff --git a/src/lib/debug.ts b/src/lib/debug.ts index b1138228da0..6f331b2e238 100644 --- a/src/lib/debug.ts +++ b/src/lib/debug.ts @@ -3,6 +3,7 @@ import { execFileSync, spawnSync } from "node:child_process"; import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { dockerExecFileSync } from "./docker/exec"; import { platform, tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; @@ -255,10 +256,9 @@ function collectDocker(collectDir: string, quick: boolean): void { // NemoClaw-labelled containers if (commandExists("docker")) { try { - const output = execFileSync( - "docker", + const output = dockerExecFileSync( ["ps", "-a", "--filter", "label=com.nvidia.nemoclaw", "--format", "{{.Names}}"], - { encoding: "utf-8", timeout: TIMEOUT_MS, stdio: ["ignore", "pipe", "ignore"] }, + { timeout: TIMEOUT_MS, stdio: ["ignore", "pipe", "ignore"] }, ); const containers = output.split("\n").filter((c) => c.trim().length > 0); for (const cid of containers) { diff --git a/src/lib/docker/container.ts b/src/lib/docker/container.ts new file mode 100644 index 00000000000..afe8d71ec3e --- /dev/null +++ b/src/lib/docker/container.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerCapture, dockerRun, type DockerCaptureOptions, type DockerRunOptions } from "./run"; + +export function dockerStop(containerName: string, opts: DockerRunOptions = {}) { + return dockerRun(["stop", containerName], opts); +} + +export function dockerRm(containerName: string, opts: DockerRunOptions = {}) { + return dockerRun(["rm", containerName], opts); +} + +export function dockerForceRm(containerName: string, opts: DockerRunOptions = {}) { + return dockerRun(["rm", "-f", containerName], opts); +} + +export function dockerRunDetached(args: readonly string[], opts: DockerRunOptions = {}) { + return dockerRun(["run", "-d", ...args], opts); +} + +export function dockerPort( + containerName: string, + containerPort: string | number, + opts: DockerCaptureOptions = {}, +): string { + return dockerCapture(["port", containerName, String(containerPort)], opts); +} + +export function dockerExecArgv(containerName: string, cmd: readonly string[]): string[] { + return ["docker", "exec", containerName, ...cmd]; +} diff --git a/src/lib/docker/exec.ts b/src/lib/docker/exec.ts new file mode 100644 index 00000000000..a5a960c406f --- /dev/null +++ b/src/lib/docker/exec.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, spawnSync, type ExecFileSyncOptionsWithStringEncoding } from "node:child_process"; + +export type DockerExecFileSyncOptions = Omit; +export type DockerSpawnSyncOptions = Parameters[2]; +export type DockerSpawnSyncResult = ReturnType; + +export function dockerExecFileSync( + args: readonly string[], + opts: DockerExecFileSyncOptions = {}, +): string { + return String(execFileSync("docker", [...args], { encoding: "utf-8", ...opts })); +} + +export function dockerSpawnSync( + args: readonly string[], + opts: DockerSpawnSyncOptions = {}, +): DockerSpawnSyncResult { + return spawnSync("docker", [...args], opts); +} diff --git a/src/lib/docker/image.ts b/src/lib/docker/image.ts new file mode 100644 index 00000000000..1de29f2f847 --- /dev/null +++ b/src/lib/docker/image.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ROOT } from "../runner"; +import { dockerCapture, dockerRun, type DockerCaptureOptions, type DockerRunOptions } from "./run"; + +export function dockerBuild( + dockerfilePath: string, + tag: string, + contextDir: string = ROOT, + opts: DockerRunOptions = {}, +) { + return dockerRun(["build", "-f", dockerfilePath, "-t", tag, contextDir], opts); +} + +export function dockerRmi(imageRef: string, opts: DockerRunOptions = {}) { + return dockerRun(["rmi", imageRef], opts); +} + +export function dockerListImagesFormat( + reference: string, + format: string, + opts: DockerCaptureOptions = {}, +): string { + return dockerCapture(["images", "--filter", `reference=${reference}`, "--format", format], opts); +} diff --git a/src/lib/docker/index.test.ts b/src/lib/docker/index.test.ts new file mode 100644 index 00000000000..0011a226c71 --- /dev/null +++ b/src/lib/docker/index.test.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const runMock = vi.fn(); +const runCaptureMock = vi.fn(); + +vi.mock("../runner", () => ({ + ROOT: "/repo/root", + run: (...args: unknown[]) => runMock(...args), + runCapture: (...args: unknown[]) => runCaptureMock(...args), +})); + +import { + dockerBuild, + dockerContainerInspectFormat, + dockerInfoFormat, + dockerListVolumesByPrefix, + dockerPull, + dockerRemoveVolumesByPrefix, + dockerRmi, + dockerRunDetached, +} from "./index"; + +describe("docker helpers", () => { + beforeEach(() => { + runMock.mockReset(); + runCaptureMock.mockReset(); + runMock.mockReturnValue({ status: 0, stdout: "", stderr: "" }); + runCaptureMock.mockReturnValue(""); + }); + + it("prefixes docker argv for pull/build/run/rmi helpers", () => { + dockerPull("ghcr.io/example/image:latest"); + dockerBuild("Dockerfile", "example:tag", "/tmp/build"); + dockerRunDetached(["--name", "example", "busybox:latest"]); + dockerRmi("example:tag"); + + expect(runMock.mock.calls).toEqual([ + [["docker", "pull", "ghcr.io/example/image:latest"], {}], + [["docker", "build", "-f", "Dockerfile", "-t", "example:tag", "/tmp/build"], {}], + [["docker", "run", "-d", "--name", "example", "busybox:latest"], {}], + [["docker", "rmi", "example:tag"], {}], + ]); + }); + + it("prefixes docker argv for info/inspect capture helpers", () => { + dockerInfoFormat("{{.KernelVersion}}", { ignoreError: true }); + dockerContainerInspectFormat("{{.State.Status}}", "example-container", { + ignoreError: true, + }); + + expect(runCaptureMock.mock.calls).toEqual([ + [["docker", "info", "--format", "{{.KernelVersion}}"], { ignoreError: true }], + [ + [ + "docker", + "inspect", + "--type", + "container", + "--format", + "{{.State.Status}}", + "example-container", + ], + { ignoreError: true }, + ], + ]); + }); + + it("filters docker volume names by exact prefix", () => { + runCaptureMock.mockReturnValue( + [ + "openshell-cluster-nemoclaw", + "openshell-cluster-nemoclaw-cache", + "openshell-cluster-nemoclaw-2", + "not-a-match", + "", + ].join("\n"), + ); + + const names = dockerListVolumesByPrefix("openshell-cluster-nemoclaw"); + + expect(names).toEqual([ + "openshell-cluster-nemoclaw", + "openshell-cluster-nemoclaw-cache", + "openshell-cluster-nemoclaw-2", + ]); + }); + + it("removes only volumes returned by the prefix probe", () => { + runCaptureMock.mockReturnValue("openshell-cluster-nemoclaw\nopenshell-cluster-nemoclaw-cache\n"); + + const removed = dockerRemoveVolumesByPrefix(" openshell-cluster-nemoclaw ", { + ignoreError: true, + }); + + expect(removed).toEqual([ + "openshell-cluster-nemoclaw", + "openshell-cluster-nemoclaw-cache", + ]); + expect(runMock).toHaveBeenCalledWith( + [ + "docker", + "volume", + "rm", + "openshell-cluster-nemoclaw", + "openshell-cluster-nemoclaw-cache", + ], + { ignoreError: true }, + ); + }); + + it("rejects empty volume prefixes", () => { + expect(() => dockerListVolumesByPrefix(" ")).toThrow(/prefix must be a non-empty string/); + expect(() => dockerRemoveVolumesByPrefix("\t")).toThrow(/prefix must be a non-empty string/); + }); + + it("treats failed volume probes as empty when ignoreError is set", () => { + runCaptureMock.mockImplementation(() => { + throw new Error("docker unavailable"); + }); + + expect(dockerRemoveVolumesByPrefix("openshell-cluster-nemoclaw", { ignoreError: true })).toEqual( + [], + ); + expect(runMock).not.toHaveBeenCalled(); + }); + + it("skips volume removal when the probe finds no matches", () => { + runCaptureMock.mockReturnValue(""); + + const removed = dockerRemoveVolumesByPrefix("openshell-cluster-nemoclaw"); + + expect(removed).toEqual([]); + expect(runMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/docker/index.ts b/src/lib/docker/index.ts new file mode 100644 index 00000000000..9e726a05775 --- /dev/null +++ b/src/lib/docker/index.ts @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export * from "./run"; +export * from "./exec"; +export * from "./pull"; +export * from "./info"; +export * from "./inspect"; +export * from "./image"; +export * from "./container"; +export * from "./volume"; +export * from "./login"; diff --git a/src/lib/docker/info.ts b/src/lib/docker/info.ts new file mode 100644 index 00000000000..fbcb92571a6 --- /dev/null +++ b/src/lib/docker/info.ts @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerCapture, type DockerCaptureOptions } from "./run"; + +export function dockerInfo(opts: DockerCaptureOptions = {}): string { + return dockerCapture(["info"], opts); +} + +export function dockerInfoFormat(format: string, opts: DockerCaptureOptions = {}): string { + return dockerCapture(["info", "--format", format], opts); +} diff --git a/src/lib/docker/inspect.ts b/src/lib/docker/inspect.ts new file mode 100644 index 00000000000..3ae7e5243dc --- /dev/null +++ b/src/lib/docker/inspect.ts @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerCapture, dockerRun, type DockerCaptureOptions, type DockerRunOptions } from "./run"; + +export function dockerInspect(args: readonly string[], opts: DockerRunOptions = {}) { + return dockerRun(["inspect", ...args], opts); +} + +export function dockerInspectFormat( + format: string, + target: string, + opts: DockerCaptureOptions = {}, +): string { + return dockerCapture(["inspect", "--format", format, target], opts); +} + +export function dockerImageInspect( + target: string, + opts: DockerRunOptions = {}, +) { + return dockerRun(["image", "inspect", target], opts); +} + +export function dockerImageInspectFormat( + format: string, + target: string, + opts: DockerCaptureOptions = {}, +): string { + return dockerCapture(["image", "inspect", "--format", format, target], opts); +} + +export function dockerContainerInspectFormat( + format: string, + containerName: string, + opts: DockerCaptureOptions = {}, +): string { + return dockerCapture(["inspect", "--type", "container", "--format", format, containerName], opts); +} diff --git a/src/lib/docker/login.ts b/src/lib/docker/login.ts new file mode 100644 index 00000000000..9548305ffd9 --- /dev/null +++ b/src/lib/docker/login.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerSpawnSync, type DockerSpawnSyncOptions, type DockerSpawnSyncResult } from "./exec"; + +export function dockerLoginPasswordStdin( + registry: string, + username: string, + password: string, + opts: DockerSpawnSyncOptions = {}, +): DockerSpawnSyncResult { + return dockerSpawnSync(["login", registry, "-u", username, "--password-stdin"], { + input: password, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + ...opts, + }); +} diff --git a/src/lib/docker/pull.ts b/src/lib/docker/pull.ts new file mode 100644 index 00000000000..5fff19417d5 --- /dev/null +++ b/src/lib/docker/pull.ts @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerRun, type DockerRunOptions } from "./run"; + +export function dockerPull(imageRef: string, opts: DockerRunOptions = {}) { + return dockerRun(["pull", imageRef], opts); +} diff --git a/src/lib/docker/run.ts b/src/lib/docker/run.ts new file mode 100644 index 00000000000..58fb786a102 --- /dev/null +++ b/src/lib/docker/run.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { run, runCapture } from "../runner"; + +export type DockerRunOptions = Parameters[1]; +export type DockerCaptureOptions = Parameters[1]; + +export function dockerArgv(args: readonly string[]): string[] { + return ["docker", ...args]; +} + +export function dockerRun(args: readonly string[], opts: DockerRunOptions = {}) { + return run(dockerArgv(args), opts); +} + +export function dockerCapture(args: readonly string[], opts: DockerCaptureOptions = {}): string { + return runCapture(dockerArgv(args), opts); +} diff --git a/src/lib/docker/volume.ts b/src/lib/docker/volume.ts new file mode 100644 index 00000000000..3ff5a7d9f95 --- /dev/null +++ b/src/lib/docker/volume.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerCapture, dockerRun, type DockerCaptureOptions, type DockerRunOptions } from "./run"; + +function splitNonEmptyLines(output: string): string[] { + return output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); +} + +function normalizeVolumePrefix(prefix: string): string { + const normalized = prefix.trim(); + if (!normalized) { + throw new Error("prefix must be a non-empty string"); + } + return normalized; +} + +export function dockerListVolumesByPrefix( + prefix: string, + opts: DockerCaptureOptions = {}, +): string[] { + const normalized = normalizeVolumePrefix(prefix); + const output = dockerCapture(["volume", "ls", "-q", "--filter", `name=${normalized}`], { + ignoreError: true, + ...opts, + }); + return splitNonEmptyLines(output).filter((name) => name.startsWith(normalized)); +} + +export function dockerRemoveVolumes(names: readonly string[], opts: DockerRunOptions = {}) { + if (names.length === 0) return null; + return dockerRun(["volume", "rm", ...names], opts); +} + +export function dockerRemoveVolumesByPrefix( + prefix: string, + opts: DockerRunOptions = {}, +): string[] { + const normalized = normalizeVolumePrefix(prefix); + let names: string[]; + try { + names = dockerListVolumesByPrefix(normalized, { + ignoreError: opts.ignoreError, + }); + } catch (error) { + if (opts.ignoreError) return []; + throw error; + } + if (names.length === 0) return names; + dockerRemoveVolumes(names, opts); + return names; +} diff --git a/src/lib/nim.ts b/src/lib/nim.ts index 01233e37c3c..c8613577645 100644 --- a/src/lib/nim.ts +++ b/src/lib/nim.ts @@ -4,7 +4,18 @@ // NIM container management — pull, start, stop, health-check NIM images. // eslint-disable-next-line @typescript-eslint/no-require-imports -const { run, runCapture } = require("./runner"); +const { runCapture } = require("./runner"); +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { + dockerContainerInspectFormat, + dockerForceRm, + dockerLoginPasswordStdin, + dockerPort, + dockerPull, + dockerRm, + dockerRunDetached, + dockerStop, +} = require("./docker"); // eslint-disable-next-line @typescript-eslint/no-require-imports const { sleepSeconds } = require("./wait"); // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -203,12 +214,7 @@ export function isNgcLoggedIn(): boolean { // NGC expects literal "$oauthtoken" as the username for API key authentication. export function dockerLoginNgc(apiKey: string): boolean { - const { spawnSync } = require("child_process"); - const result = spawnSync("docker", ["login", "nvcr.io", "-u", "$oauthtoken", "--password-stdin"], { - input: apiKey, - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], - }); + const result = dockerLoginPasswordStdin("nvcr.io", "$oauthtoken", apiKey); if (result.error) { console.error(` Docker error: ${result.error.message}`); return false; @@ -226,7 +232,7 @@ export function pullNimImage(model: string): string { process.exit(1); } console.log(` Pulling NIM image: ${image}`); - run(["docker", "pull", image]); + dockerPull(image); return image; } @@ -242,14 +248,18 @@ export function startNimContainerByName(name: string, model: string, port = VLLM process.exit(1); } - run(["docker", "rm", "-f", name], { ignoreError: true }); + dockerForceRm(name, { ignoreError: true }); console.log(` Starting NIM container: ${name}`); - run([ - "docker", "run", "-d", "--gpus", "all", - "-p", `${Number(port)}:8000`, - "--name", name, - "--shm-size", "16g", + dockerRunDetached([ + "--gpus", + "all", + "-p", + `${Number(port)}:8000`, + "--name", + name, + "--shm-size", + "16g", image, ]); return name; @@ -302,8 +312,8 @@ export function stopNimContainerByName( ): void { if (!silent) console.log(` Stopping NIM container: ${name}`); const stdio = silent ? ["ignore", "ignore", "ignore"] : undefined; - run(["docker", "stop", name], { ignoreError: true, ...(stdio && { stdio }) }); - run(["docker", "rm", name], { ignoreError: true, ...(stdio && { stdio }) }); + dockerStop(name, { ignoreError: true, ...(stdio && { stdio }) }); + dockerRm(name, { ignoreError: true, ...(stdio && { stdio }) }); } export function nimStatus(sandboxName: string, port?: number): NimStatus { @@ -313,19 +323,14 @@ export function nimStatus(sandboxName: string, port?: number): NimStatus { export function nimStatusByName(name: string, port?: number): NimStatus { try { - const state = runCapture( - ["docker", "inspect", "--format", "{{.State.Status}}", name], - { ignoreError: true }, - ); + const state = dockerContainerInspectFormat("{{.State.Status}}", name, { ignoreError: true }); if (!state) return { running: false, container: name }; let healthy = false; if (state === "running") { let resolvedHostPort = port != null ? Number(port) : 0; if (!resolvedHostPort) { - const mapping = runCapture(["docker", "port", name, "8000"], { - ignoreError: true, - }); + const mapping = dockerPort(name, "8000", { ignoreError: true }); const m = mapping && mapping.match(/:(\d+)\s*$/); resolvedHostPort = m ? Number(m[1]) : VLLM_PORT; } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 36b8e4c1bbd..f7e534be19a 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -27,6 +27,21 @@ const LOCAL_INFERENCE_TIMEOUT_SECS = envInt("NEMOCLAW_LOCAL_INFERENCE_TIMEOUT", const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; const runner: typeof import("./runner") = require("./runner"); const { ROOT, SCRIPTS, redact, run, runShell, runCapture, runFile, shellQuote, validateName } = runner; +const docker: typeof import("./docker") = require("./docker"); +const { + dockerContainerInspectFormat, + dockerExecArgv, + dockerImageInspect, + dockerImageInspectFormat, + dockerInfo, + dockerInfoFormat, + dockerInspect, + dockerPull, + dockerRemoveVolumesByPrefix, + dockerRm, + dockerRmi, + dockerStop, +} = docker; const errnoUtils: typeof import("./errno") = require("./errno"); const { isErrnoException } = errnoUtils; @@ -242,8 +257,8 @@ const BACK_TO_SELECTION = "__NEMOCLAW_BACK_TO_SELECTION__"; */ function verifyGatewayContainerRunning() { const containerName = `openshell-cluster-${GATEWAY_NAME}`; - const result = run( - ["docker", "inspect", "--type", "container", "--format", "{{.State.Running}}", containerName], + const result = dockerInspect( + ["--type", "container", "--format", "{{.State.Running}}", containerName], { ignoreError: true, suppressOutput: true }, ); if (result.status === 0 && String(result.stdout || "").trim() === "true") { @@ -597,22 +612,16 @@ const SANDBOX_BASE_TAG = "latest"; */ function pullAndResolveBaseImageDigest(): { digest: string; ref: string } | null { const imageWithTag = `${SANDBOX_BASE_IMAGE}:${SANDBOX_BASE_TAG}`; - try { - run(["docker", "pull", imageWithTag], { suppressOutput: true }); - } catch { + const pullResult = dockerPull(imageWithTag, { ignoreError: true, suppressOutput: true }); + if (pullResult.status !== 0) { // Pull failed — caller should fall back to unpin :latest return null; } - let inspectOutput; - try { - inspectOutput = runCapture( - ["docker", "inspect", "--format", "{{json .RepoDigests}}", imageWithTag], - { ignoreError: false }, - ); - } catch { - return null; - } + const inspectOutput = dockerImageInspectFormat("{{json .RepoDigests}}", imageWithTag, { + ignoreError: true, + }); + if (!inspectOutput) return null; // RepoDigests is a JSON array like ["ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:abc..."]. // Filter to the entry matching our registry — index ordering is not guaranteed. @@ -1825,7 +1834,7 @@ function getResumeConfigConflicts( } function getContainerRuntime(): ContainerRuntime { - const info = runCapture(["docker", "info"], { ignoreError: true }); + const info = dockerInfo({ ignoreError: true }); return inferContainerRuntime(info); } @@ -1922,21 +1931,14 @@ function destroyGateway() { } // openshell gateway destroy doesn't remove Docker volumes, which leaves // corrupted cluster state that breaks the next gateway start. Clean them up. - removeGatewayClusterVolumes(); + dockerRemoveVolumesByPrefix(`openshell-cluster-${GATEWAY_NAME}`, { ignoreError: true }); } function getGatewayClusterContainerState(): string { const containerName = getGatewayClusterContainerName(); - const state = runCapture( - [ - "docker", - "inspect", - "--type", - "container", - "--format", - "{{.State.Status}}{{if .State.Health}} {{.State.Health.Status}}{{end}}", - containerName, - ], + const state = dockerContainerInspectFormat( + "{{.State.Status}}{{if .State.Health}} {{.State.Health.Status}}{{end}}", + containerName, { ignoreError: true }, ) .trim() @@ -1969,25 +1971,7 @@ function getGatewayClusterContainerName(): string { } function buildGatewayClusterExecArgv(script: string): string[] { - return ["docker", "exec", getGatewayClusterContainerName(), "sh", "-lc", script]; -} - -function getGatewayClusterVolumeNames(): string[] { - return runCapture(["docker", "volume", "ls", "-q", "--filter", `name=${getGatewayClusterContainerName()}`], { - ignoreError: true, - }) - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); -} - -function removeGatewayClusterVolumes(opts: { suppressOutput?: boolean } = {}): void { - const names = getGatewayClusterVolumeNames(); - if (names.length === 0) return; - run(["docker", "volume", "rm", ...names], { - ignoreError: true, - ...(opts.suppressOutput ? { suppressOutput: true } : {}), - }); + return dockerExecArgv(getGatewayClusterContainerName(), ["sh", "-lc", script]); } function hostCommandExists(commandName: string): boolean { @@ -2561,26 +2545,29 @@ async function preflight(): Promise> { // OpenShell has no metadata for it (gatewayReuseState === "missing"). if (gatewayReuseState === "missing") { const containerName = `openshell-cluster-${GATEWAY_NAME}`; - const inspectResult = run( - ["docker", "inspect", "--type", "container", "--format", "{{.State.Status}}", containerName], + const inspectResult = dockerInspect( + ["--type", "container", "--format", "{{.State.Status}}", containerName], { ignoreError: true, suppressOutput: true }, ); if (inspectResult.status === 0) { console.log(" Cleaning up orphaned gateway container..."); - run(["docker", "stop", containerName], { + dockerStop(containerName, { ignoreError: true, suppressOutput: true, }); - run(["docker", "rm", containerName], { + dockerRm(containerName, { ignoreError: true, suppressOutput: true, }); - const postInspectResult = run(["docker", "inspect", "--type", "container", containerName], { + const postInspectResult = dockerInspect(["--type", "container", containerName], { ignoreError: true, suppressOutput: true, }); if (postInspectResult.status !== 0) { - removeGatewayClusterVolumes({ suppressOutput: true }); + dockerRemoveVolumesByPrefix(`openshell-cluster-${GATEWAY_NAME}`, { + ignoreError: true, + suppressOutput: true, + }); registry.clearAll(); console.log(" ✓ Orphaned gateway container removed"); } else { @@ -3542,7 +3529,7 @@ async function createSandbox( // Destroy old sandbox and clean up its host-side Docker image. runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); if (previousEntry?.imageTag) { - const rmiResult = run(["docker", "rmi", previousEntry.imageTag], { + const rmiResult = dockerRmi(previousEntry.imageTag, { ignoreError: true, suppressOutput: true, }); @@ -3744,11 +3731,11 @@ async function createSandbox( // Check if the image exists locally before falling back to unpinned :latest. // On a first-time install behind a firewall with no cached image, warn early // so the user knows the build will likely fail. - const localCheck = runCapture( - ["docker", "image", "inspect", `${SANDBOX_BASE_IMAGE}:${SANDBOX_BASE_TAG}`], - { ignoreError: true }, - ); - if (localCheck) { + const localCheck = dockerImageInspect(`${SANDBOX_BASE_IMAGE}:${SANDBOX_BASE_TAG}`, { + ignoreError: true, + suppressOutput: true, + }); + if (localCheck.status === 0) { console.warn(" Warning: could not pull base image from registry; using cached :latest."); } else { console.warn( @@ -4029,7 +4016,7 @@ async function createSandbox( try { if (process.platform === "darwin") { - const vmKernel = runCapture(["docker", "info", "--format", "{{.KernelVersion}}"], { + const vmKernel = dockerInfoFormat("{{.KernelVersion}}", { ignoreError: true, }).trim(); if (vmKernel) { diff --git a/src/lib/sandbox-config.ts b/src/lib/sandbox-config.ts index e80d95888ad..527d60c8311 100644 --- a/src/lib/sandbox-config.ts +++ b/src/lib/sandbox-config.ts @@ -16,8 +16,8 @@ const readline = require("readline"); const fs = require("fs"); const os = require("os"); const path = require("path"); -const { execFileSync } = require("child_process"); const { validateName } = require("./runner"); +const { dockerExecFileSync } = require("./docker/exec"); const credentialFilter: typeof import("./credential-filter") = require("./credential-filter"); const { stripCredentials, isConfigObject, isConfigValue } = credentialFilter; const { appendAuditEntry } = require("./shields-audit"); @@ -516,8 +516,7 @@ async function configSet(sandboxName: string, opts: ConfigSetOpts = {}): Promise // Write config to sandbox via kubectl exec (bypasses Landlock) console.log(` Writing config to sandbox (${target.configPath})...`); const content = fs.readFileSync(tmpFile, "utf-8"); - execFileSync( - "docker", + dockerExecFileSync( [ "exec", "-i", @@ -540,8 +539,7 @@ async function configSet(sandboxName: string, opts: ConfigSetOpts = {}): Promise // Fix ownership via kubectl exec (bypasses Landlock) try { - execFileSync( - "docker", + dockerExecFileSync( [ "exec", K3S_CONTAINER, diff --git a/src/lib/shields.ts b/src/lib/shields.ts index 79b920064fb..fd5565641de 100644 --- a/src/lib/shields.ts +++ b/src/lib/shields.ts @@ -11,8 +11,9 @@ const fs = require("fs"); const path = require("path"); -const { fork, execFileSync } = require("child_process"); +const { fork } = require("child_process"); const { run, runCapture, validateName, shellQuote } = require("./runner"); +const { dockerExecFileSync } = require("./docker/exec"); const { buildPolicyGetCommand, buildPolicySetCommand, @@ -36,46 +37,34 @@ const STATE_DIR = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "state"); const K3S_CONTAINER = "openshell-cluster-nemoclaw"; +function kubectlExecArgv(sandboxName: string, cmd: string[]): string[] { + return [ + "exec", + K3S_CONTAINER, + "kubectl", + "exec", + "-n", + "openshell", + sandboxName, + "-c", + "agent", + "--", + ...cmd, + ]; +} + function kubectlExec(sandboxName: string, cmd: string[]): void { - execFileSync( - "docker", - [ - "exec", - K3S_CONTAINER, - "kubectl", - "exec", - "-n", - "openshell", - sandboxName, - "-c", - "agent", - "--", - ...cmd, - ], - { stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - ); + dockerExecFileSync(kubectlExecArgv(sandboxName, cmd), { + stdio: ["ignore", "pipe", "pipe"], + timeout: 15000, + }); } function kubectlExecCapture(sandboxName: string, cmd: string[]): string { - return execFileSync( - "docker", - [ - "exec", - K3S_CONTAINER, - "kubectl", - "exec", - "-n", - "openshell", - sandboxName, - "-c", - "agent", - "--", - ...cmd, - ], - { stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - ) - .toString() - .trim(); + return dockerExecFileSync(kubectlExecArgv(sandboxName, cmd), { + stdio: ["ignore", "pipe", "pipe"], + timeout: 15000, + }).trim(); } // Re-export for tests and external consumers diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index be94180868c..18dbf13d83c 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -21,14 +21,13 @@ const R = _useColor ? "\x1b[0m" : ""; const _RD = _useColor ? "\x1b[1;31m" : ""; const YW = _useColor ? "\x1b[1;33m" : ""; +const { ROOT, run, runInteractive, shellQuote, validateName } = require("./lib/runner"); const { - ROOT, - run, - runCapture: _runCapture, - runInteractive, - shellQuote, - validateName, -} = require("./lib/runner"); + dockerCapture, + dockerListImagesFormat, + dockerRemoveVolumesByPrefix, + dockerRmi, +} = require("./lib/docker"); const { resolveOpenshell } = require("./lib/resolve-openshell"); const { fetchGatewayAuthTokenFromSandbox, @@ -167,25 +166,15 @@ function captureOpenshell(args: CommandArgs, opts: RunnerOptions = {}) { }); } -function removeGatewayClusterVolumes(): void { - const prefix = `openshell-cluster-${NEMOCLAW_GATEWAY_NAME}`; - const names = _runCapture(["docker", "volume", "ls", "-q", "--filter", `name=${prefix}`], { - ignoreError: true, - }) - .split(/\r?\n/) - .map((line: string) => line.trim()) - .filter((line: string) => line.startsWith(prefix)); - if (names.length === 0) return; - run(["docker", "volume", "rm", ...names], { ignoreError: true }); -} - function cleanupGatewayAfterLastSandbox() { runOpenshell(["forward", "stop", DASHBOARD_FORWARD_PORT], { ignoreError: true, stdio: ["ignore", "ignore", "ignore"], }); runOpenshell(["gateway", "destroy", "-g", NEMOCLAW_GATEWAY_NAME], { ignoreError: true }); - removeGatewayClusterVolumes(); + dockerRemoveVolumesByPrefix(`openshell-cluster-${NEMOCLAW_GATEWAY_NAME}`, { + ignoreError: true, + }); } function hasNoLiveSandboxes() { @@ -2529,7 +2518,7 @@ function cleanupSandboxServices( function removeSandboxImage(sandboxName: string) { const sb = registry.getSandbox(sandboxName); if (!sb?.imageTag) return; - const result = run(["docker", "rmi", sb.imageTag], { ignoreError: true }); + const result = dockerRmi(sb.imageTag, { ignoreError: true }); if (result.status === 0) { console.log(` Removed Docker image ${sb.imageTag}`); } else { @@ -3267,8 +3256,7 @@ function renderSnapshotTable( function resolveSrcPodImage(srcName: string): string | null { const gatewayContainer = `openshell-cluster-${NEMOCLAW_GATEWAY_NAME}`; try { - const result = spawnSync( - "docker", + const output = dockerCapture( [ "exec", gatewayContainer, @@ -3281,10 +3269,9 @@ function resolveSrcPodImage(srcName: string): string | null { "-o", 'jsonpath={.spec.containers[?(@.name=="agent")].image}', ], - { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: 10000 }, + { ignoreError: true, timeout: 10000 }, ); - if (result.status !== 0) return null; - const img = (result.stdout || "").trim().split(/\s+/)[0]; + const img = output.trim().split(/\s+/)[0]; return img || null; } catch { return null; @@ -3644,23 +3631,18 @@ async function garbageCollectImages(args: string[] = []): Promise { const skipConfirm = args.includes("--yes") || args.includes("--force"); // 1. List all openshell/sandbox-from images on the host - const imagesResult = spawnSync( - "docker", - [ - "images", - "--filter", - "reference=openshell/sandbox-from", - "--format", + let imagesOutput = ""; + try { + imagesOutput = dockerListImagesFormat( + "openshell/sandbox-from", "{{.Repository}}:{{.Tag}}\t{{.Size}}", - ], - { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"] }, - ); - if (imagesResult.status !== 0) { + ); + } catch { console.error(" Failed to query Docker images. Is Docker running?"); process.exit(1); } - const allImages = (imagesResult.stdout || "") + const allImages = imagesOutput .split("\n") .map((line: string) => line.trim()) .filter(Boolean) @@ -3716,9 +3698,11 @@ async function garbageCollectImages(args: string[] = []): Promise { let removed = 0; let failed = 0; for (const img of orphans) { - const rmiResult = spawnSync("docker", ["rmi", img.tag], { + const rmiResult = dockerRmi(img.tag, { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], + ignoreError: true, + suppressOutput: true, }); if (rmiResult.status === 0) { console.log(` ${G}✓${R} Removed ${img.tag}`); diff --git a/test/cli.test.ts b/test/cli.test.ts index 832b0d38126..cfeba520ab8 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -579,7 +579,7 @@ describe("CLI dispatch", () => { const localBin = path.join(home, "bin"); const registryDir = path.join(home, ".nemoclaw"); const openshellLog = path.join(home, "openshell.log"); - const bashLog = path.join(home, "bash.log"); + const bashLog = path.join(home, "docker.log"); fs.mkdirSync(localBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync( @@ -641,7 +641,7 @@ describe("CLI dispatch", () => { const localBin = path.join(home, "bin"); const registryDir = path.join(home, ".nemoclaw"); const openshellLog = path.join(home, "openshell.log"); - const bashLog = path.join(home, "bash.log"); + const bashLog = path.join(home, "docker.log"); fs.mkdirSync(localBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync( @@ -712,7 +712,7 @@ describe("CLI dispatch", () => { const localBin = path.join(home, "bin"); const registryDir = path.join(home, ".nemoclaw"); const openshellLog = path.join(home, "openshell.log"); - const bashLog = path.join(home, "bash.log"); + const bashLog = path.join(home, "docker.log"); fs.mkdirSync(localBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync( @@ -833,7 +833,7 @@ describe("CLI dispatch", () => { const localBin = path.join(home, "bin"); const registryDir = path.join(home, ".nemoclaw"); const openshellLog = path.join(home, "openshell.log"); - const bashLog = path.join(home, "bash.log"); + const bashLog = path.join(home, "docker.log"); fs.mkdirSync(localBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync( @@ -907,7 +907,7 @@ describe("CLI dispatch", () => { const localBin = path.join(home, "bin"); const registryDir = path.join(home, ".nemoclaw"); const openshellLog = path.join(home, "openshell.log"); - const bashLog = path.join(home, "bash.log"); + const bashLog = path.join(home, "docker.log"); fs.mkdirSync(localBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync( diff --git a/test/docker-abstraction-guard.test.ts b/test/docker-abstraction-guard.test.ts new file mode 100644 index 00000000000..8673ab18cb3 --- /dev/null +++ b/test/docker-abstraction-guard.test.ts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = path.join(import.meta.dirname, ".."); +const TSX = path.join(REPO_ROOT, "node_modules", ".bin", "tsx"); +const INVENTORY_SCRIPT = path.join(REPO_ROOT, "scripts", "list-command-helper-uses.ts"); +const DOCKER_ABSTRACTION_PREFIX = "src/lib/docker/"; + +type CommandUse = { + filePath: string; + line: number; + column: number; + kind: "call" | "assign"; + name: string; + expression: string; + commandHead: string | null; + snippet: string; +}; + +function listCommandUses(): CommandUse[] { + const result = spawnSync( + TSX, + [INVENTORY_SCRIPT, "--json", "--list-calls", "src"], + { + cwd: REPO_ROOT, + encoding: "utf-8", + }, + ); + expect(result.status).toBe(0); + return JSON.parse(result.stdout) as CommandUse[]; +} + +describe("Docker command abstraction guard", () => { + it("keeps direct docker process launches inside src/lib/docker", () => { + const directDockerCalls = listCommandUses().filter( + (entry) => + entry.kind === "call" && + entry.commandHead === "docker" && + !entry.filePath.startsWith(DOCKER_ABSTRACTION_PREFIX), + ); + + if (directDockerCalls.length > 0) { + const formatted = directDockerCalls + .map( + (entry) => + ` - ${entry.filePath}:${String(entry.line)}:${String(entry.column)} ${entry.expression}: ${entry.snippet}`, + ) + .join("\n"); + throw new Error( + `Direct docker process launches must use the src/lib/docker abstractions.\n` + + `Do not add calls like run(["docker", ...]), runCapture(["docker", ...]), ` + + `spawnSync("docker", ...), or execFileSync("docker", ...) outside ${DOCKER_ABSTRACTION_PREFIX}.\n` + + `Add or reuse a helper under ${DOCKER_ABSTRACTION_PREFIX} (for example dockerRun(), dockerCapture(), dockerPull(), dockerRmi(), dockerExecFileSync(), or dockerSpawnSync()) and call that instead.\n` + + `Violations:\n${formatted}`, + ); + } + + expect(directDockerCalls).toEqual([]); + }); +}); diff --git a/test/gateway-cleanup.test.ts b/test/gateway-cleanup.test.ts index fefb3683a73..20dd13eddbd 100644 --- a/test/gateway-cleanup.test.ts +++ b/test/gateway-cleanup.test.ts @@ -16,9 +16,11 @@ const ROOT = path.resolve(import.meta.dirname, ".."); describe("gateway cleanup: Docker volumes removed on failure (#17)", () => { it("onboard.js: destroyGateway() removes Docker volumes", () => { const content = fs.readFileSync(path.join(ROOT, "src/lib/onboard.ts"), "utf-8"); - expect(content).toContain("removeGatewayClusterVolumes"); - expect(content).toContain('"docker", "volume", "ls", "-q"'); - expect(content).toContain("openshell-cluster"); + const match = content.match(/function destroyGateway\([\s\S]*?^}/m); + expect(match).toBeTruthy(); + if (!match) throw new Error("Expected destroyGateway() in src/lib/onboard.ts"); + expect(match[0]).toContain("dockerRemoveVolumesByPrefix"); + expect(match[0]).toContain("openshell-cluster"); }); it("onboard.js: volume cleanup runs on gateway start failure", () => { diff --git a/test/gateway-liveness-probe.test.ts b/test/gateway-liveness-probe.test.ts index 41ac125a22f..b69096bc664 100644 --- a/test/gateway-liveness-probe.test.ts +++ b/test/gateway-liveness-probe.test.ts @@ -18,11 +18,13 @@ describe("gateway liveness probe (#2020)", () => { const content = fs.readFileSync(path.join(ROOT, "src/lib/onboard.ts"), "utf-8"); it("verifyGatewayContainerRunning() helper exists and checks Docker state", () => { - expect(content).toContain("function verifyGatewayContainerRunning()"); - // Must use docker inspect to probe container state - expect(content).toMatch(/"docker",\s*"inspect",\s*"--type",\s*"container"/); + const match = content.match(/function verifyGatewayContainerRunning\([\s\S]*?^}/m); + expect(match).toBeTruthy(); + if (!match) throw new Error("Expected verifyGatewayContainerRunning() in src/lib/onboard.ts"); + // Must use a Docker inspect helper to probe container state + expect(match[0]).toMatch(/docker(?:Inspect|ContainerInspectFormat)\(/); // Must check .State.Running, not just container existence - expect(content).toContain("{{.State.Running}}"); + expect(match[0]).toContain("{{.State.Running}}"); }); it("preflight probes the container when gatewayReuseState is 'healthy'", () => { diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index 4be41daa146..1e541247d00 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -14,8 +14,10 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { const nemoclawSrc = fs.readFileSync(path.join(ROOT, "src/nemoclaw.ts"), "utf-8"); it("removeSandboxImage() helper exists and calls docker rmi", () => { - expect(nemoclawSrc).toContain("function removeSandboxImage("); - expect(nemoclawSrc).toMatch(/docker.*rmi/); + const match = nemoclawSrc.match(/function removeSandboxImage[\s\S]*?^}/m); + expect(match).toBeTruthy(); + if (!match) throw new Error("Expected removeSandboxImage() in src/nemoclaw.ts"); + expect(match[0]).toMatch(/dockerRmi\(|docker.*\.rmi\(/); }); it("sandboxDestroy calls removeSandboxImage before registry.removeSandbox", () => { @@ -77,8 +79,10 @@ describe("image cleanup: onboard records imageTag in registry (#2086)", () => { it("onboard recreate path cleans up old image", () => { // When recreating, the old image should be removed - expect(onboardSrc).toMatch(/previousEntry\?\.imageTag/); - expect(onboardSrc).toMatch(/docker.*rmi.*previousEntry\.imageTag/); + const match = onboardSrc.match(/if \(previousEntry\?\.imageTag\)[\s\S]*?^\s*}/m); + expect(match).toBeTruthy(); + if (!match) throw new Error("Expected previousEntry image cleanup block in src/lib/onboard.ts"); + expect(match[0]).toMatch(/dockerRmi\(|docker.*\.rmi\(/); }); });