diff --git a/.github/workflows/pr-self-hosted.yaml b/.github/workflows/pr-self-hosted.yaml index 43e1af94e7b..c6d136a6431 100644 --- a/.github/workflows/pr-self-hosted.yaml +++ b/.github/workflows/pr-self-hosted.yaml @@ -127,6 +127,15 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + - name: Download image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -134,7 +143,15 @@ jobs: path: /tmp - name: Load image - run: gunzip -c /tmp/isolation-image.tar.gz | docker load + run: | + gunzip -c /tmp/isolation-image.tar.gz | docker load + docker image inspect nemoclaw-production >/dev/null + + - name: Run glibc probe lifecycle regression + env: + NEMOCLAW_RUN_GLIBC_PROBE_DOCKER_E2E: "1" + NEMOCLAW_TEST_IMAGE: nemoclaw-production + run: npx vitest run --project integration test/image-compatibility-docker-lifecycle.test.ts --silent=false --reporter=default - name: Run gateway isolation E2E tests run: NEMOCLAW_TEST_IMAGE=nemoclaw-production bash test/e2e-gateway-isolation.sh diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 478ae1b8007..ec2277f0995 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -613,6 +613,10 @@ jobs: steps: - *checkout + - *setup-node + + - *install-root-dependencies + - &download-isolation-image name: Download image artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -627,6 +631,12 @@ jobs: gunzip -c /tmp/isolation-image.tar.gz | docker load docker image inspect nemoclaw-production >/dev/null + - name: Run glibc probe lifecycle regression + env: + NEMOCLAW_RUN_GLIBC_PROBE_DOCKER_E2E: "1" + NEMOCLAW_TEST_IMAGE: nemoclaw-production + run: npx vitest run --project integration test/image-compatibility-docker-lifecycle.test.ts --silent=false --reporter=default + - name: Run gateway isolation E2E tests run: NEMOCLAW_TEST_IMAGE=nemoclaw-production bash test/e2e-gateway-isolation.sh diff --git a/src/lib/agent/base-image-hermes-resolution.test.ts b/src/lib/agent/base-image-hermes-resolution.test.ts index 46da1b1805e..22fa469cd0a 100644 --- a/src/lib/agent/base-image-hermes-resolution.test.ts +++ b/src/lib/agent/base-image-hermes-resolution.test.ts @@ -99,7 +99,7 @@ describe("Hermes base-image resolver integration", () => { (inspectOutputByKey.get(`${format}\0${ref}`) ?? "").trim(), ); dockerMocks.capture.mockImplementation( - (args: string[]) => captureByEntrypoint.get(args[3]) ?? "", + (args: string[]) => captureByEntrypoint.get(args[args.indexOf("--entrypoint") + 1]) ?? "", ); }); diff --git a/src/lib/sandbox-base-image/image-compatibility.test.ts b/src/lib/sandbox-base-image/image-compatibility.test.ts index 471fccf1e42..9063b541b2b 100644 --- a/src/lib/sandbox-base-image/image-compatibility.test.ts +++ b/src/lib/sandbox-base-image/image-compatibility.test.ts @@ -5,10 +5,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ dockerCapture: vi.fn(), + dockerForceRm: vi.fn(), })); vi.mock("../adapters/docker", () => ({ dockerCapture: mocks.dockerCapture, + dockerForceRm: mocks.dockerForceRm, })); import { @@ -21,6 +23,10 @@ import { describe("sandbox base-image glibc compatibility", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.dockerCapture.mockImplementation((args: readonly string[]) => + args[0] === "run" ? "ldd (GNU libc) 2.41" : "", + ); + mocks.dockerForceRm.mockReturnValue({ error: undefined, status: 0 }); }); it.each([ @@ -44,21 +50,145 @@ describe("sandbox base-image glibc compatibility", () => { }); it("reads the image glibc version through the Docker adapter", () => { - mocks.dockerCapture.mockReturnValue("ldd (GNU libc) 2.41\nCopyright notice"); + mocks.dockerCapture.mockImplementation((args: readonly string[]) => + args[0] === "run" ? "ldd (GNU libc) 2.41\nCopyright notice" : "", + ); expect(getImageGlibcVersion("nemoclaw:test")).toBe("2.41"); expect(mocks.dockerCapture).toHaveBeenCalledWith( - ["run", "--rm", "--entrypoint", "/usr/bin/ldd", "nemoclaw:test", "--version"], + [ + "run", + "--rm", + "--name", + expect.stringMatching(/^nemoclaw-glibc-probe-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/), + "--entrypoint", + "/usr/bin/ldd", + "nemoclaw:test", + "--version", + ], { ignoreError: true, timeout: 20_000 }, ); }); + it("retries a probe with missing output and removes its retained container (#8375)", () => { + let probeCount = 0; + mocks.dockerCapture.mockImplementation((args: readonly string[]) => { + if (args[0] !== "run") return ""; + probeCount += 1; + return probeCount === 1 ? "" : "ldd (Debian GLIBC 2.41-12+deb13u3) 2.41"; + }); + + expect(getImageGlibcVersion("nemoclaw:cold")).toBe("2.41"); + + const probeCalls = mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run"); + expect(probeCalls.map((call) => call[1]?.timeout)).toEqual([20_000, 120_000]); + const containerNames = probeCalls.map((call) => call[0]?.[3]); + expect(new Set(containerNames)).toHaveProperty("size", 2); + expect(mocks.dockerForceRm).toHaveBeenCalledWith(containerNames[0], { + ignoreError: true, + suppressOutput: true, + timeout: 20_000, + }); + expect(mocks.dockerCapture).toHaveBeenCalledWith( + [ + "container", + "ls", + "--all", + "--filter", + `name=^/${containerNames[0]}$`, + "--format", + "{{.Names}}", + ], + { timeout: 20_000 }, + ); + }); + + it("removes both retained containers when both probe attempts return no output (#8375)", () => { + mocks.dockerCapture.mockReturnValue(""); + + expect(getImageGlibcVersion("nemoclaw:cold")).toBeNull(); + + const probeCalls = mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run"); + expect(probeCalls.map((call) => call[1]?.timeout)).toEqual([20_000, 120_000]); + const containerNames = probeCalls.map((call) => call[0]?.[3]); + expect(new Set(containerNames)).toHaveProperty("size", 2); + expect(mocks.dockerForceRm.mock.calls).toEqual( + containerNames.map((containerName) => [ + containerName, + { ignoreError: true, suppressOutput: true, timeout: 20_000 }, + ]), + ); + const absenceChecks = mocks.dockerCapture.mock.calls.filter( + (call) => call[0]?.[0] === "container", + ); + expect(absenceChecks).toHaveLength(2); + }); + + it("accepts a failed removal only when the retained container is already absent (#8375)", () => { + let probeCount = 0; + mocks.dockerForceRm.mockReturnValue({ error: undefined, status: 1 }); + mocks.dockerCapture.mockImplementation((args: readonly string[]) => { + if (args[0] !== "run") return ""; + probeCount += 1; + return probeCount === 1 ? "" : "ldd (GNU libc) 2.41"; + }); + + expect(getImageGlibcVersion("nemoclaw:cold")).toBe("2.41"); + expect(probeCount).toBe(2); + }); + + it.each([ + [{ error: undefined, status: 1 }, "returned status 1"], + [ + { error: new Error("Docker removal failed"), status: null }, + "failed before returning an exit status", + ], + ])("stops before retry when cleanup %s leaves the retained container present (#8375)", (removal, expectedStatus) => { + let retainedContainerName = ""; + mocks.dockerForceRm.mockReturnValue(removal); + mocks.dockerCapture.mockImplementation((args: readonly string[]) => { + if (args[0] === "run") { + retainedContainerName = String(args[3]); + return ""; + } + return args[0] === "container" ? retainedContainerName : ""; + }); + + expect(() => getImageGlibcVersion("nemoclaw:cold")).toThrow( + new RegExp(`cleanup ${expectedStatus}; container nemoclaw-glibc-probe-.+ is still present`), + ); + expect(mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run")).toHaveLength(1); + }); + + it("stops before retry when retained-container absence cannot be verified (#8375)", () => { + mocks.dockerCapture.mockImplementation((args: readonly string[]) => { + if (args[0] === "run") return ""; + throw new Error("Docker daemon unavailable during cleanup verification"); + }); + + expect(() => getImageGlibcVersion("nemoclaw:cold")).toThrow( + "Docker daemon unavailable during cleanup verification", + ); + expect(mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run")).toHaveLength(1); + }); + + it("does not retry non-empty incompatible output", () => { + mocks.dockerCapture.mockImplementation((args: readonly string[]) => + args[0] === "run" ? "musl libc (x86_64)\nVersion 1.2.5" : "", + ); + + expect(getImageGlibcVersion("nemoclaw:musl")).toBeNull(); + expect(mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run")).toHaveLength(1); + }); + it.each([ ["ldd (GNU libc) 2.41", "2.39", { ok: true, version: "2.41" }], ["ldd (GNU libc) 2.36", "2.39", { ok: false, version: "2.36" }], ["musl libc (x86_64)\nVersion 1.2.5", "2.39", { ok: false, version: null }], ])("enforces the minimum glibc version %#", (output, minimum, expected) => { - mocks.dockerCapture.mockReturnValue(output); + mocks.dockerCapture.mockImplementation((args: readonly string[]) => + args[0] === "run" ? output : "", + ); expect(imageMeetsMinimumGlibc("nemoclaw:test", minimum)).toEqual(expected); }); diff --git a/src/lib/sandbox-base-image/image-compatibility.ts b/src/lib/sandbox-base-image/image-compatibility.ts index a8ff8505ae0..840923cd301 100644 --- a/src/lib/sandbox-base-image/image-compatibility.ts +++ b/src/lib/sandbox-base-image/image-compatibility.ts @@ -1,9 +1,37 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerCapture } from "../adapters/docker"; +import { randomUUID } from "node:crypto"; +import { dockerCapture, dockerForceRm } from "../adapters/docker"; import { OPENSHELL_SANDBOX_MIN_GLIBC } from "./types"; +const GLIBC_PROBE_TIMEOUTS_MS = [20_000, 120_000] as const; +const GLIBC_PROBE_CLEANUP_TIMEOUT_MS = 20_000; + +function removeRetainedGlibcProbe(containerName: string): void { + const removal = dockerForceRm(containerName, { + ignoreError: true, + suppressOutput: true, + timeout: GLIBC_PROBE_CLEANUP_TIMEOUT_MS, + }); + const remainingNames = dockerCapture( + ["container", "ls", "--all", "--filter", `name=^/${containerName}$`, "--format", "{{.Names}}"], + { timeout: GLIBC_PROBE_CLEANUP_TIMEOUT_MS }, + ) + .split(/\r?\n/) + .map((name) => name.trim()) + .filter(Boolean); + + if (remainingNames.includes(containerName)) { + const removalStatus = removal.error + ? "failed before returning an exit status" + : `returned status ${String(removal.status)}`; + throw new Error( + `Docker glibc probe cleanup ${removalStatus}; container ${containerName} is still present`, + ); + } +} + export function parseGlibcVersion(output: string | null | undefined): string | null { const text = String(output || ""); const firstLine = text.split(/\r?\n/).find((line) => line.trim()); @@ -30,11 +58,31 @@ export function versionGte(left = "0.0.0", right = "0.0.0"): boolean { } export function getImageGlibcVersion(imageRef: string): string | null { - const output = dockerCapture( - ["run", "--rm", "--entrypoint", "/usr/bin/ldd", imageRef, "--version"], - { ignoreError: true, timeout: 20_000 }, - ); - return parseGlibcVersion(output); + for (const timeout of GLIBC_PROBE_TIMEOUTS_MS) { + const containerName = `nemoclaw-glibc-probe-${randomUUID()}`; + let output = ""; + try { + output = dockerCapture( + [ + "run", + "--rm", + "--name", + containerName, + "--entrypoint", + "/usr/bin/ldd", + imageRef, + "--version", + ], + { ignoreError: true, timeout }, + ); + } finally { + if (!output) { + removeRetainedGlibcProbe(containerName); + } + } + if (output) return parseGlibcVersion(output); + } + return null; } export function imageMeetsMinimumGlibc( diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index d603e096667..39f283f0ffc 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -153,7 +153,11 @@ function mockOnboardRunCapture(command, options = {}) { if (isOpenClawSecurityInventoryProbe(command)) { return "nemoclaw-security-inventory-ok"; } - if (/^docker run --rm --entrypoint \/usr\/bin\/ldd \S+ --version$/.test(normalized)) { + if ( + normalized.startsWith("docker run ") && + normalized.includes(" --entrypoint /usr/bin/ldd ") && + normalized.endsWith(" --version") + ) { return "ldd (GNU libc) 2.41"; } return mockSandboxExecCurl(command, options); diff --git a/test/image-compatibility-docker-lifecycle.test.ts b/test/image-compatibility-docker-lifecycle.test.ts new file mode 100644 index 00000000000..23ac1fbf6e9 --- /dev/null +++ b/test/image-compatibility-docker-lifecycle.test.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { imageMeetsMinimumGlibc } from "../src/lib/sandbox-base-image/image-compatibility.js"; +import { testTimeoutOptions } from "./helpers/timeouts.js"; + +const RUN_DOCKER_E2E = process.env.NEMOCLAW_RUN_GLIBC_PROBE_DOCKER_E2E === "1"; +const TEST_IMAGE = process.env.NEMOCLAW_TEST_IMAGE ?? "nemoclaw-production"; + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function cleanupProbeContainers(realDocker: string, probeNamesPath: string): void { + [...new Set(fs.readFileSync(probeNamesPath, "utf8").trim().split("\n").filter(Boolean))].forEach( + (probeName) => spawnSync(realDocker, ["rm", "-f", probeName], { stdio: "ignore" }), + ); +} + +describe.runIf(RUN_DOCKER_E2E)("sandbox base-image glibc Docker lifecycle", () => { + it( + "removes a retained first probe before accepting the retry (#8375)", + testTimeoutOptions(150_000), + () => { + const realDocker = execFileSync("which", ["docker"], { encoding: "utf8" }).trim(); + execFileSync(realDocker, ["image", "inspect", TEST_IMAGE], { stdio: "ignore" }); + + const fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-glibc-probe-")); + const shimPath = path.join(fixtureDir, "docker"); + const firstProbeNamePath = path.join(fixtureDir, "first-probe-name"); + const probeNamesPath = path.join(fixtureDir, "probe-names"); + const markerPath = path.join(fixtureDir, "first-probe-created"); + const logPath = path.join(fixtureDir, "docker-shim.log"); + const hadOriginalPath = Object.hasOwn(process.env, "PATH"); + const originalPath = process.env.PATH ?? ""; + let firstProbeName = ""; + + fs.writeFileSync(probeNamesPath, ""); + + const shim = `#!/usr/bin/env bash +set -euo pipefail +real_docker=${shellQuote(realDocker)} +test_image=${shellQuote(TEST_IMAGE)} +marker=${shellQuote(markerPath)} +name_file=${shellQuote(firstProbeNamePath)} +probe_names_file=${shellQuote(probeNamesPath)} +log_file=${shellQuote(logPath)} + +if [[ "\${1:-}" == "run" ]]; then + probe_name="" + for ((index = 1; index <= \$#; index += 1)); do + if [[ "\${!index}" == "--name" ]]; then + name_index=\$((index + 1)) + probe_name="\${!name_index}" + break + fi + done + printf '%s\n' "\$probe_name" >>"\$probe_names_file" + if [[ ! -e "\$marker" ]]; then + : >"\$marker" + printf '%s\n' "\$probe_name" >"\$name_file" + printf 'retained %s\n' "\$probe_name" >>"\$log_file" + "\$real_docker" create --name "\$probe_name" --entrypoint /usr/bin/ldd "\$test_image" --version >/dev/null + exit 124 + fi + printf 'retried %s\n' "\$probe_name" >>"\$log_file" +elif [[ "\${1:-}" == "rm" && "\${2:-}" == "-f" ]]; then + printf 'removed %s\n' "\${3:-}" >>"\$log_file" +fi + +exec "\$real_docker" "\$@" +`; + + fs.writeFileSync(shimPath, shim, { mode: 0o755 }); + process.env.PATH = `${fixtureDir}:${originalPath}`; + + try { + expect(imageMeetsMinimumGlibc(TEST_IMAGE, "2.17")).toEqual({ + ok: true, + version: expect.stringMatching(/^\d+(?:\.\d+)+$/), + }); + + firstProbeName = fs.readFileSync(firstProbeNamePath, "utf8").trim(); + expect(firstProbeName).toMatch(/^nemoclaw-glibc-probe-/); + expect( + spawnSync(realDocker, ["container", "inspect", firstProbeName], { + stdio: "ignore", + }).status, + ).not.toBe(0); + const lifecycleLog = fs.readFileSync(logPath, "utf8").trim().split("\n"); + expect(lifecycleLog).toEqual([ + `retained ${firstProbeName}`, + `removed ${firstProbeName}`, + expect.stringMatching(/^retried nemoclaw-glibc-probe-/), + ]); + expect(lifecycleLog[2]).not.toBe(`retried ${firstProbeName}`); + } finally { + Reflect.deleteProperty(process.env, "PATH"); + [originalPath] + .filter(() => hadOriginalPath) + .forEach((savedPath) => Reflect.set(process.env, "PATH", savedPath)); + cleanupProbeContainers(realDocker, probeNamesPath); + fs.rmSync(fixtureDir, { recursive: true, force: true }); + } + }, + ); +});