fix(sandbox): retry cold glibc probes

.github/workflows/pr-self-hosted.yaml

@@ -127,14 +127,31 @@ jobs:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      - name: Set up Node
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22.19.0
+          cache: npm
+
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
+
       - name: Download image artifact
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: isolation-image
           path: /tmp
 
       - name: Load image
-        run: gunzip -c /tmp/isolation-image.tar.gz | docker load
+        run: |
+          gunzip -c /tmp/isolation-image.tar.gz | docker load
+          docker image inspect nemoclaw-production >/dev/null
+
+      - name: Run glibc probe lifecycle regression
+        env:
+          NEMOCLAW_RUN_GLIBC_PROBE_DOCKER_E2E: "1"
+          NEMOCLAW_TEST_IMAGE: nemoclaw-production
+        run: npx vitest run --project integration test/image-compatibility-docker-lifecycle.test.ts --silent=false --reporter=default
 
       - name: Run gateway isolation E2E tests
         run: NEMOCLAW_TEST_IMAGE=nemoclaw-production bash test/e2e-gateway-isolation.sh

.github/workflows/sandbox-images-and-e2e.yaml

@@ -613,6 +613,10 @@ jobs:
     steps:
       - *checkout
 
+      - *setup-node
+
+      - *install-root-dependencies
+
       - &download-isolation-image
         name: Download image artifact
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
@@ -627,6 +631,12 @@ jobs:
           gunzip -c /tmp/isolation-image.tar.gz | docker load
           docker image inspect nemoclaw-production >/dev/null
 
+      - name: Run glibc probe lifecycle regression
+        env:
+          NEMOCLAW_RUN_GLIBC_PROBE_DOCKER_E2E: "1"
+          NEMOCLAW_TEST_IMAGE: nemoclaw-production
+        run: npx vitest run --project integration test/image-compatibility-docker-lifecycle.test.ts --silent=false --reporter=default
+
       - name: Run gateway isolation E2E tests
         run: NEMOCLAW_TEST_IMAGE=nemoclaw-production bash test/e2e-gateway-isolation.sh
 

src/lib/agent/base-image-hermes-resolution.test.ts

@@ -99,7 +99,7 @@ describe("Hermes base-image resolver integration", () => {
       (inspectOutputByKey.get(`${format}\0${ref}`) ?? "").trim(),
     );
     dockerMocks.capture.mockImplementation(
-      (args: string[]) => captureByEntrypoint.get(args[3]) ?? "",
+      (args: string[]) => captureByEntrypoint.get(args[args.indexOf("--entrypoint") + 1]) ?? "",
     );
   });
 

src/lib/sandbox-base-image/image-compatibility.test.ts

@@ -5,10 +5,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 
 const mocks = vi.hoisted(() => ({
   dockerCapture: vi.fn(),
+  dockerForceRm: vi.fn(),
 }));
 
 vi.mock("../adapters/docker", () => ({
   dockerCapture: mocks.dockerCapture,
+  dockerForceRm: mocks.dockerForceRm,
 }));
 
 import {
@@ -21,6 +23,10 @@ import {
 describe("sandbox base-image glibc compatibility", () => {
   beforeEach(() => {
     vi.clearAllMocks();
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) =>
+      args[0] === "run" ? "ldd (GNU libc) 2.41" : "",
+    );
+    mocks.dockerForceRm.mockReturnValue({ error: undefined, status: 0 });
   });
 
   it.each([
@@ -44,21 +50,145 @@ describe("sandbox base-image glibc compatibility", () => {
   });
 
   it("reads the image glibc version through the Docker adapter", () => {
-    mocks.dockerCapture.mockReturnValue("ldd (GNU libc) 2.41\nCopyright notice");
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) =>
+      args[0] === "run" ? "ldd (GNU libc) 2.41\nCopyright notice" : "",
+    );
 
     expect(getImageGlibcVersion("nemoclaw:test")).toBe("2.41");
     expect(mocks.dockerCapture).toHaveBeenCalledWith(
-      ["run", "--rm", "--entrypoint", "/usr/bin/ldd", "nemoclaw:test", "--version"],
+      [
+        "run",
+        "--rm",
+        "--name",
+        expect.stringMatching(/^nemoclaw-glibc-probe-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/),
+        "--entrypoint",
+        "/usr/bin/ldd",
+        "nemoclaw:test",
+        "--version",
+      ],
       { ignoreError: true, timeout: 20_000 },
     );
   });
 
+  it("retries a probe with missing output and removes its retained container (#8375)", () => {
+    let probeCount = 0;
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) => {
+      if (args[0] !== "run") return "";
+      probeCount += 1;
+      return probeCount === 1 ? "" : "ldd (Debian GLIBC 2.41-12+deb13u3) 2.41";
+    });
+
+    expect(getImageGlibcVersion("nemoclaw:cold")).toBe("2.41");
+
+    const probeCalls = mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run");
+    expect(probeCalls.map((call) => call[1]?.timeout)).toEqual([20_000, 120_000]);
+    const containerNames = probeCalls.map((call) => call[0]?.[3]);
+    expect(new Set(containerNames)).toHaveProperty("size", 2);
+    expect(mocks.dockerForceRm).toHaveBeenCalledWith(containerNames[0], {
+      ignoreError: true,
+      suppressOutput: true,
+      timeout: 20_000,
+    });
+    expect(mocks.dockerCapture).toHaveBeenCalledWith(
+      [
+        "container",
+        "ls",
+        "--all",
+        "--filter",
+        `name=^/${containerNames[0]}$`,
+        "--format",
+        "{{.Names}}",
+      ],
+      { timeout: 20_000 },
+    );
+  });
+
+  it("removes both retained containers when both probe attempts return no output (#8375)", () => {
+    mocks.dockerCapture.mockReturnValue("");
+
+    expect(getImageGlibcVersion("nemoclaw:cold")).toBeNull();
+
+    const probeCalls = mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run");
+    expect(probeCalls.map((call) => call[1]?.timeout)).toEqual([20_000, 120_000]);
+    const containerNames = probeCalls.map((call) => call[0]?.[3]);
+    expect(new Set(containerNames)).toHaveProperty("size", 2);
+    expect(mocks.dockerForceRm.mock.calls).toEqual(
+      containerNames.map((containerName) => [
+        containerName,
+        { ignoreError: true, suppressOutput: true, timeout: 20_000 },
+      ]),
+    );
+    const absenceChecks = mocks.dockerCapture.mock.calls.filter(
+      (call) => call[0]?.[0] === "container",
+    );
+    expect(absenceChecks).toHaveLength(2);
+  });
+
+  it("accepts a failed removal only when the retained container is already absent (#8375)", () => {
+    let probeCount = 0;
+    mocks.dockerForceRm.mockReturnValue({ error: undefined, status: 1 });
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) => {
+      if (args[0] !== "run") return "";
+      probeCount += 1;
+      return probeCount === 1 ? "" : "ldd (GNU libc) 2.41";
+    });
+
+    expect(getImageGlibcVersion("nemoclaw:cold")).toBe("2.41");
+    expect(probeCount).toBe(2);
+  });
+
+  it.each([
+    [{ error: undefined, status: 1 }, "returned status 1"],
+    [
+      { error: new Error("Docker removal failed"), status: null },
+      "failed before returning an exit status",
+    ],
+  ])("stops before retry when cleanup %s leaves the retained container present (#8375)", (removal, expectedStatus) => {
+    let retainedContainerName = "";
+    mocks.dockerForceRm.mockReturnValue(removal);
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) => {
+      if (args[0] === "run") {
+        retainedContainerName = String(args[3]);
+        return "";
+      }
+      return args[0] === "container" ? retainedContainerName : "";
+    });
+
+    expect(() => getImageGlibcVersion("nemoclaw:cold")).toThrow(
+      new RegExp(`cleanup ${expectedStatus}; container nemoclaw-glibc-probe-.+ is still present`),
+    );
+    expect(mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run")).toHaveLength(1);
+  });
+
+  it("stops before retry when retained-container absence cannot be verified (#8375)", () => {
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) => {
+      if (args[0] === "run") return "";
+      throw new Error("Docker daemon unavailable during cleanup verification");
+    });
+
+    expect(() => getImageGlibcVersion("nemoclaw:cold")).toThrow(
+      "Docker daemon unavailable during cleanup verification",
+    );
+    expect(mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run")).toHaveLength(1);
+  });
+
+  it("does not retry non-empty incompatible output", () => {
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) =>
+      args[0] === "run" ? "musl libc (x86_64)\nVersion 1.2.5" : "",
+    );
+
+    expect(getImageGlibcVersion("nemoclaw:musl")).toBeNull();
+    expect(mocks.dockerCapture.mock.calls.filter((call) => call[0]?.[0] === "run")).toHaveLength(1);
+  });
+
   it.each([
     ["ldd (GNU libc) 2.41", "2.39", { ok: true, version: "2.41" }],
     ["ldd (GNU libc) 2.36", "2.39", { ok: false, version: "2.36" }],
     ["musl libc (x86_64)\nVersion 1.2.5", "2.39", { ok: false, version: null }],
   ])("enforces the minimum glibc version %#", (output, minimum, expected) => {
-    mocks.dockerCapture.mockReturnValue(output);
+    mocks.dockerCapture.mockImplementation((args: readonly string[]) =>
+      args[0] === "run" ? output : "",
+    );
 
     expect(imageMeetsMinimumGlibc("nemoclaw:test", minimum)).toEqual(expected);
   });

src/lib/sandbox-base-image/image-compatibility.ts

@@ -1,9 +1,37 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { dockerCapture } from "../adapters/docker";
+import { randomUUID } from "node:crypto";
+import { dockerCapture, dockerForceRm } from "../adapters/docker";
 import { OPENSHELL_SANDBOX_MIN_GLIBC } from "./types";
 
+const GLIBC_PROBE_TIMEOUTS_MS = [20_000, 120_000] as const;
+const GLIBC_PROBE_CLEANUP_TIMEOUT_MS = 20_000;
+
+function removeRetainedGlibcProbe(containerName: string): void {
+  const removal = dockerForceRm(containerName, {
+    ignoreError: true,
+    suppressOutput: true,
+    timeout: GLIBC_PROBE_CLEANUP_TIMEOUT_MS,
+  });
+  const remainingNames = dockerCapture(
+    ["container", "ls", "--all", "--filter", `name=^/${containerName}$`, "--format", "{{.Names}}"],
+    { timeout: GLIBC_PROBE_CLEANUP_TIMEOUT_MS },
+  )
+    .split(/\r?\n/)
+    .map((name) => name.trim())
+    .filter(Boolean);
+
+  if (remainingNames.includes(containerName)) {
+    const removalStatus = removal.error
+      ? "failed before returning an exit status"
+      : `returned status ${String(removal.status)}`;
+    throw new Error(
+      `Docker glibc probe cleanup ${removalStatus}; container ${containerName} is still present`,
+    );
+  }
+}
+
 export function parseGlibcVersion(output: string | null | undefined): string | null {
   const text = String(output || "");
   const firstLine = text.split(/\r?\n/).find((line) => line.trim());
@@ -30,11 +58,31 @@ export function versionGte(left = "0.0.0", right = "0.0.0"): boolean {
 }
 
 export function getImageGlibcVersion(imageRef: string): string | null {
-  const output = dockerCapture(
-    ["run", "--rm", "--entrypoint", "/usr/bin/ldd", imageRef, "--version"],
-    { ignoreError: true, timeout: 20_000 },
-  );
-  return parseGlibcVersion(output);
+  for (const timeout of GLIBC_PROBE_TIMEOUTS_MS) {
+    const containerName = `nemoclaw-glibc-probe-${randomUUID()}`;
+    let output = "";
+    try {
+      output = dockerCapture(
+        [
+          "run",
+          "--rm",
+          "--name",
+          containerName,
+          "--entrypoint",
+          "/usr/bin/ldd",
+          imageRef,
+          "--version",
+        ],
+        { ignoreError: true, timeout },
+      );
+    } finally {
+      if (!output) {
+        removeRetainedGlibcProbe(containerName);
+      }
+    }
+    if (output) return parseGlibcVersion(output);
+  }
+  return null;
 }
 
 export function imageMeetsMinimumGlibc(

test/helpers/onboard-script-mocks.cjs

@@ -153,7 +153,11 @@ function mockOnboardRunCapture(command, options = {}) {
   if (isOpenClawSecurityInventoryProbe(command)) {
     return "nemoclaw-security-inventory-ok";
   }
-  if (/^docker run --rm --entrypoint \/usr\/bin\/ldd \S+ --version$/.test(normalized)) {
+  if (
+    normalized.startsWith("docker run ") &&
+    normalized.includes(" --entrypoint /usr/bin/ldd ") &&
+    normalized.endsWith(" --version")
+  ) {
     return "ldd (GNU libc) 2.41";
   }
   return mockSandboxExecCurl(command, options);