From bbf46c172b1d8080d97ef68c52cf46a78aa6662c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 07:02:42 -0700 Subject: [PATCH 01/19] fix(security): trust corporate CA before Hermes final-stage fetches Signed-off-by: Carlos Villela --- agents/hermes/Dockerfile | 89 +++++++++++-------- ci/source-shape-test-budget.json | 5 ++ .../security/configure-corporate-ca-trust.mdx | 4 +- test/corporate-ca-build-tls-anchor.test.ts | 63 +++++++++++++ 4 files changed, 125 insertions(+), 36 deletions(-) diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 616f4e5ad9f..9deece23072 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -150,24 +150,70 @@ COPY scripts/checks/node-tar-image-scan.mts /scripts/checks/node-tar-image-scan. # hadolint ignore=DL3006 FROM ${BASE_IMAGE} +# Base64-encoded host corporate-proxy CA bundle (#6210). Empty by default. When +# onboard detects an operator-supplied corporate CA on the host it bakes it +# here; the RUN below decodes it to a root-owned file that the entrypoint +# appends to the OpenShell trust bundle at runtime. The CA is a public +# certificate, not a secret, so baking it into an image layer is acceptable. +ARG NEMOCLAW_CORPORATE_CA_B64 + +# Decode the host corporate-proxy CA (#6210) to a root-owned, read-only file +# when onboard baked one in. No-op when NEMOCLAW_CORPORATE_CA_B64 is empty. The +# ARG is expanded by the shell (not interpolated into source), and its value is +# base64 sanitized host-side, so this is not an injection vector. Must run as +# root, before the USER sandbox drop below. +# hadolint ignore=DL3059,DL4006 +RUN if [ -n "${NEMOCLAW_CORPORATE_CA_B64}" ]; then \ + command -v base64 >/dev/null 2>&1 || { echo "[nemoclaw] base64 is required to decode NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image" >&2; exit 1; }; \ + command -v openssl >/dev/null 2>&1 || { echo "[nemoclaw] openssl is required to validate NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image (#6210)" >&2; exit 1; }; \ + mkdir -p /usr/local/share/nemoclaw \ + && { printf '%s' "${NEMOCLAW_CORPORATE_CA_B64}" | base64 --decode > /tmp/nemoclaw-corporate-ca.decoded 2>/dev/null \ + || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1; }; } \ + && awk '/-----BEGIN CERTIFICATE-----/{f=1} f{print} /-----END CERTIFICATE-----/{f=0}' /tmp/nemoclaw-corporate-ca.decoded > /usr/local/share/nemoclaw/corporate-ca.pem \ + && rm -f /tmp/nemoclaw-corporate-ca.decoded \ + && { grep -qF -- "-----BEGIN CERTIFICATE-----" /usr/local/share/nemoclaw/corporate-ca.pem || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ + && { openssl crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem >/dev/null 2>&1 || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ + && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem \ + && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem \ + && echo "[nemoclaw] baked host corporate-proxy CA into image trust (#6210)"; \ + fi + +# Use the decoded CA for Node.js package operations in this final stage. Node.js +# ignores the path when no CA was baked. At runtime, nemoclaw-start replaces it +# with the merged OpenShell and corporate bundle. +ENV NODE_EXTRA_CA_CERTS=/usr/local/share/nemoclaw/corporate-ca.pem + # Cross-stage root copies are accepted by Docker's legacy builder and create # one final-image layer while preserving metadata on existing parent paths. COPY --from=hermes-npm-patch-payload / / # The final Hermes image owns the shipped dependency boundary independently of -# base freshness. Reassert the idempotent npm-private node-tar fix here. -RUN node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts \ - --npm-root /usr/local/lib/node_modules/npm +# base freshness. Reassert the idempotent npm-private node-tar fix here. When +# onboarding supplied a corporate CA, use it for the registry-backed download. +RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \ + export CURL_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \ + fi; \ + node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts \ + --npm-root /usr/local/lib/node_modules/npm # Reassert the npm-private brace-expansion fix for the exact final filesystem. +# When onboarding supplied a corporate CA, use it for the registry-backed +# download. # hadolint ignore=DL3059 -RUN node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts \ - --npm-root /usr/local/lib/node_modules/npm +RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \ + export CURL_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \ + fi; \ + node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts \ + --npm-root /usr/local/lib/node_modules/npm -# Reassert the npm-private ip-address fix for the exact final filesystem. +# Reassert the npm-private ip-address fix for the exact final filesystem. When +# onboarding supplied a corporate CA, use it for the registry-backed download. # hadolint ignore=DL3059 -RUN node --experimental-strip-types /scripts/lib/patch-bundled-npm-ip-address.mts \ - --npm-root /usr/local/lib/node_modules/npm +RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \ + export CURL_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \ + fi; \ + node --experimental-strip-types /scripts/lib/patch-bundled-npm-ip-address.mts \ + --npm-root /usr/local/lib/node_modules/npm # Keep the final image contract explicit even when the published base image # changes independently of this Dockerfile. @@ -605,12 +651,6 @@ ARG NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER=0 ARG NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64=W10= ARG NEMOCLAW_BUILD_ID=default ARG NEMOCLAW_DARWIN_VM_COMPAT=0 -# Base64-encoded host corporate-proxy CA bundle (#6210). Empty by default. When -# onboard detects an operator-supplied corporate CA on the host it bakes it -# here; the RUN below decodes it to a root-owned file that the entrypoint -# appends to the OpenShell trust bundle at runtime. The CA is a public -# certificate, not a secret, so baking it into an image layer is acceptable. -ARG NEMOCLAW_CORPORATE_CA_B64 # Total model context window (input + output tokens). Empty by default so # Hermes auto-detects from the endpoint's /v1/models max_model_len; onboard # rewrites this ARG (via dockerfile-patch) when it probes a runtime value or @@ -667,27 +707,6 @@ RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-b --agent hermes --phase managed-image-capability-union; \ fi -# Decode the host corporate-proxy CA (#6210) to a root-owned, read-only file -# when onboard baked one in. No-op when NEMOCLAW_CORPORATE_CA_B64 is empty. The -# ARG is expanded by the shell (not interpolated into source), and its value is -# base64 sanitized host-side, so this is not an injection vector. Must run as -# root, before the USER sandbox drop below. -# hadolint ignore=DL3059,DL4006 -RUN if [ -n "${NEMOCLAW_CORPORATE_CA_B64}" ]; then \ - command -v base64 >/dev/null 2>&1 || { echo "[nemoclaw] base64 is required to decode NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image" >&2; exit 1; }; \ - command -v openssl >/dev/null 2>&1 || { echo "[nemoclaw] openssl is required to validate NEMOCLAW_CORPORATE_CA_B64 but is not installed in the build image (#6210)" >&2; exit 1; }; \ - mkdir -p /usr/local/share/nemoclaw \ - && { printf '%s' "${NEMOCLAW_CORPORATE_CA_B64}" | base64 --decode > /tmp/nemoclaw-corporate-ca.decoded 2>/dev/null \ - || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 is not valid base64; expected a single-line base64-encoded PEM (#6210)" >&2; exit 1; }; } \ - && awk '/-----BEGIN CERTIFICATE-----/{f=1} f{print} /-----END CERTIFICATE-----/{f=0}' /tmp/nemoclaw-corporate-ca.decoded > /usr/local/share/nemoclaw/corporate-ca.pem \ - && rm -f /tmp/nemoclaw-corporate-ca.decoded \ - && { grep -qF -- "-----BEGIN CERTIFICATE-----" /usr/local/share/nemoclaw/corporate-ca.pem || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ - && { openssl crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem >/dev/null 2>&1 || { echo "[nemoclaw] NEMOCLAW_CORPORATE_CA_B64 did not decode to a bundle of valid X.509 certificates (#6210)" >&2; exit 1; }; } \ - && chown root:root /usr/local/share/nemoclaw/corporate-ca.pem \ - && chmod 0444 /usr/local/share/nemoclaw/corporate-ca.pem \ - && echo "[nemoclaw] baked host corporate-proxy CA into image trust (#6210)"; \ - fi - WORKDIR /sandbox USER sandbox diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index e4594c5a309..2fa0c2c31f6 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -91,6 +91,11 @@ "test": "trusts the corporate CA before the DCode discovery runtime npm install", "category": "security" }, + { + "file": "test/corporate-ca-build-tls-anchor.test.ts", + "test": "uses the corporate CA conditionally for all Hermes registry remediations", + "category": "security" + }, { "file": "test/dcode-base-image-workflow.test.ts", "test": "accepts every discovered publisher and rejects supply-chain mutations", diff --git a/docs/security/configure-corporate-ca-trust.mdx b/docs/security/configure-corporate-ca-trust.mdx index 201d9808465..a55b5e5c30c 100644 --- a/docs/security/configure-corporate-ca-trust.mdx +++ b/docs/security/configure-corporate-ca-trust.mdx @@ -39,7 +39,9 @@ It sets `NODE_EXTRA_CA_CERTS` before build-time Node.js dependency verification, -The Hermes Dockerfile decodes the bundle after its managed build-time dependency steps, so the corporate CA does not apply to those earlier operations. +The Hermes discovery-runtime installer applies the corporate CA before its npm operations. +The final Hermes image stage decodes the CA immediately after `FROM ${BASE_IMAGE}` and sets `NODE_EXTRA_CA_CERTS` before later npm operations. +The registry-backed npm remediations set `CURL_CA_BUNDLE` before each download only when the decoded certificate file exists. diff --git a/test/corporate-ca-build-tls-anchor.test.ts b/test/corporate-ca-build-tls-anchor.test.ts index 1a4ca5174b6..20c4982be76 100644 --- a/test/corporate-ca-build-tls-anchor.test.ts +++ b/test/corporate-ca-build-tls-anchor.test.ts @@ -173,3 +173,66 @@ describe("DCode corporate proxy CA cold-build trust (#8119)", () => { expect(curlAnchorIndex).toBeLessThan(ipAddressPatchIndex); }); }); + +describe("Hermes corporate proxy CA final-stage trust", () => { + const dockerfile = readFileSync( + join(import.meta.dirname, "../agents/hermes/Dockerfile"), + "utf-8", + ); + + // source-shape-contract: security -- Hermes final-stage registry clients must trust the decoded corporate CA before making HTTPS requests + it("uses the corporate CA conditionally for all Hermes registry remediations", () => { + const finalFromIndex = dockerfile.indexOf("FROM ${BASE_IMAGE}"); + const finalStage = dockerfile.slice(finalFromIndex); + const argIndex = finalStage.indexOf("ARG NEMOCLAW_CORPORATE_CA_B64"); + const decodeIndex = finalStage.indexOf( + 'RUN if [ -n "${NEMOCLAW_CORPORATE_CA_B64}" ]; then', + argIndex, + ); + const nodeAnchorIndex = finalStage.indexOf( + "ENV NODE_EXTRA_CA_CERTS=/usr/local/share/nemoclaw/corporate-ca.pem", + decodeIndex, + ); + const payloadCopyIndex = finalStage.indexOf( + "COPY --from=hermes-npm-patch-payload / /", + nodeAnchorIndex, + ); + const conditionalCurlTrust = `RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \\ + export CURL_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \\ + fi; \\`; + const remediationCommands = [ + "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", + "node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts", + "node --experimental-strip-types /scripts/lib/patch-bundled-npm-ip-address.mts", + ]; + const npmCommandIndexes = [...finalStage.matchAll(/^\s*npm\s+(?:ci|run)\b/gmu)].map( + (match) => match.index, + ); + + for (const [name, index] of Object.entries({ + finalFromIndex, + argIndex, + decodeIndex, + nodeAnchorIndex, + payloadCopyIndex, + })) { + expect(index, name).toBeGreaterThan(-1); + } + expect(argIndex).toBeLessThan(decodeIndex); + expect(decodeIndex).toBeLessThan(nodeAnchorIndex); + expect(nodeAnchorIndex).toBeLessThan(payloadCopyIndex); + for (const remediationCommand of remediationCommands) { + const remediationIndex = finalStage.indexOf(remediationCommand, payloadCopyIndex); + expect(remediationIndex, remediationCommand).toBeGreaterThan(payloadCopyIndex); + const runIndex = finalStage.lastIndexOf("\nRUN ", remediationIndex) + 1; + expect(runIndex, remediationCommand).toBeGreaterThan(payloadCopyIndex); + expect(finalStage.slice(runIndex, remediationIndex).trim(), remediationCommand).toBe( + conditionalCurlTrust, + ); + } + expect(npmCommandIndexes.length).toBeGreaterThan(0); + for (const npmCommandIndex of npmCommandIndexes) { + expect(nodeAnchorIndex).toBeLessThan(npmCommandIndex); + } + }); +}); From 3c0df7be014c21abc2f7ec8df3d0a85def5fdfb4 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 08:12:46 -0700 Subject: [PATCH 02/19] test(hermes): accept guarded npm remediation runs Signed-off-by: Carlos Villela --- ...-npm-brace-expansion-dockerfile-contract.test.ts | 4 ++-- ...ndled-npm-ip-address-dockerfile-contract.test.ts | 4 ++-- test/hermes-final-image-layout.test.ts | 2 +- test/node-tar-dockerfile-contract.test.ts | 13 ++++--------- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts b/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts index 5cb814cc124..76dcf3b2d51 100644 --- a/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts +++ b/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts @@ -27,7 +27,7 @@ const finalDockerfiles = [ const copyInstruction = "COPY scripts/patch-bundled-npm-brace-expansion.mts /scripts/patch-bundled-npm-brace-expansion.mts"; const patchInstruction = - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts"; + "node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts"; describe("bundled npm brace-expansion image remediation contract", () => { it("binds the replacement to the reviewed npm and registry artifact", () => { @@ -60,7 +60,7 @@ describe("bundled npm brace-expansion image remediation contract", () => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); const tarPatch = source.indexOf( - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", + "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", ); const bracePatch = source.indexOf(patchInstruction); diff --git a/test/bundled-npm-ip-address-dockerfile-contract.test.ts b/test/bundled-npm-ip-address-dockerfile-contract.test.ts index 67309121327..a5d55ea9281 100644 --- a/test/bundled-npm-ip-address-dockerfile-contract.test.ts +++ b/test/bundled-npm-ip-address-dockerfile-contract.test.ts @@ -66,10 +66,10 @@ describe("bundled npm ip-address image remediation contract", () => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); const tarPatch = source.indexOf( - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", + "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", ); const bracePatch = source.indexOf( - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts", + "node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts", ); const ipAddressPatch = source.indexOf(patchCommand); diff --git a/test/hermes-final-image-layout.test.ts b/test/hermes-final-image-layout.test.ts index 93867396d2b..48cc7967219 100644 --- a/test/hermes-final-image-layout.test.ts +++ b/test/hermes-final-image-layout.test.ts @@ -319,7 +319,7 @@ describe("Hermes final image layout", () => { const scan = indexOfRequired(finalStage, scanCopy); const tarPatch = indexOfRequired( finalStage, - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", + "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", ); const certifiInstall = indexOfRequired(finalStage, "RUN _hermes_certifi="); const agentChmod = indexOfRequired( diff --git a/test/node-tar-dockerfile-contract.test.ts b/test/node-tar-dockerfile-contract.test.ts index 80ee6da1289..a00d870a50d 100644 --- a/test/node-tar-dockerfile-contract.test.ts +++ b/test/node-tar-dockerfile-contract.test.ts @@ -29,6 +29,7 @@ const dockerfiles = [ installsWithNpm: false, }, ] as const; +const patchCommand = "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts"; function completedStage(source: string): string { const finalStageStart = [...source.matchAll(/^FROM\b/gmu)].at(-1)?.index; @@ -61,9 +62,7 @@ describe("node-tar image remediation contract", () => { ])("installs curl before patching the bundled npm tar in $file", (file) => { const source = completedStage(fs.readFileSync(path.join(repoRoot, file), "utf8")); const curlInstall = source.indexOf("curl="); - const patchRun = source.indexOf( - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", - ); + const patchRun = source.indexOf(patchCommand); expect(curlInstall, file).toBeGreaterThanOrEqual(0); expect(patchRun, file).toBeGreaterThan(curlInstall); @@ -90,9 +89,7 @@ describe("node-tar image remediation contract", () => { const patchCopy = patchInputStage.indexOf( "COPY scripts/patch-bundled-npm-tar.mts /scripts/patch-bundled-npm-tar.mts", ); - const patchRun = source.indexOf( - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", - ); + const patchRun = source.indexOf(patchCommand); const scanCopy = scanInputStage.indexOf( "COPY scripts/checks/node-tar-image-scan.mts /scripts/checks/node-tar-image-scan.mts", ); @@ -151,9 +148,7 @@ describe("reviewed npm image remediation contract", () => { { file: "agents/langchain-deepagents-code/Dockerfile.base", installsWithNpm: false }, ])("upgrades npm before use in $file", ({ file, installsWithNpm }) => { const source = completedStage(fs.readFileSync(path.join(repoRoot, file), "utf8")); - const patchRun = source.indexOf( - "RUN node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", - ); + const patchRun = source.indexOf(patchCommand); const upgradeCopy = source.indexOf( "COPY scripts/upgrade-bundled-npm.mts /scripts/upgrade-bundled-npm.mts", ); From 75c9e34a11b021019e8fe022824fc8a21b479307 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 08:42:48 -0700 Subject: [PATCH 03/19] test(security): validate Dockerfile remediation runs Signed-off-by: Carlos Villela --- ...race-expansion-dockerfile-contract.test.ts | 27 ++-- ...npm-ip-address-dockerfile-contract.test.ts | 36 ++--- test/dockerfile-run-commands.test.ts | 54 +++++++ test/helpers/dockerfile-run-commands.ts | 150 ++++++++++++++++++ test/hermes-final-image-layout.test.ts | 5 +- test/node-tar-dockerfile-contract.test.ts | 19 ++- 6 files changed, 251 insertions(+), 40 deletions(-) create mode 100644 test/dockerfile-run-commands.test.ts create mode 100644 test/helpers/dockerfile-run-commands.ts diff --git a/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts b/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts index 76dcf3b2d51..5aefb4eb99d 100644 --- a/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts +++ b/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts @@ -12,6 +12,7 @@ import { REVIEWED_NPM_VERSION, } from "../scripts/patch-bundled-npm-brace-expansion.mts"; import { REVIEWED_NPM_VERSION as UPGRADED_NPM_VERSION } from "../scripts/upgrade-bundled-npm.mts"; +import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; const repoRoot = path.resolve(import.meta.dirname, ".."); const baseDockerfiles = [ @@ -43,15 +44,16 @@ describe("bundled npm brace-expansion image remediation contract", () => { it.each(baseDockerfiles)("patches the reviewed npm tree after upgrading it in %s", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const upgrade = source.indexOf( - "RUN node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", - ); - const patch = source.indexOf(patchInstruction); + const upgrade = requireSingleDockerfileRunCommand( + source, + "node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", + ).commandStart; + const patch = requireSingleDockerfileRunCommand(source, patchInstruction); expect(copy, file).toBeGreaterThanOrEqual(0); expect(upgrade, file).toBeGreaterThan(copy); - expect(patch, file).toBeGreaterThan(upgrade); - expect(source.slice(patch)).toContain("--npm-root /usr/local/lib/node_modules/npm"); + expect(patch.commandStart, file).toBeGreaterThan(upgrade); + expect(patch.instruction.text, file).toContain("--npm-root /usr/local/lib/node_modules/npm"); }); it.each( @@ -59,14 +61,17 @@ describe("bundled npm brace-expansion image remediation contract", () => { )("reasserts the private package fix in the completed %s filesystem", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const tarPatch = source.indexOf( + const tarPatch = requireSingleDockerfileRunCommand( + source, "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", - ); - const bracePatch = source.indexOf(patchInstruction); + ).commandStart; + const bracePatch = requireSingleDockerfileRunCommand(source, patchInstruction); expect(copy, file).toBeGreaterThanOrEqual(0); expect(tarPatch, file).toBeGreaterThan(copy); - expect(bracePatch, file).toBeGreaterThan(tarPatch); - expect(source.slice(bracePatch)).toContain("--npm-root /usr/local/lib/node_modules/npm"); + expect(bracePatch.commandStart, file).toBeGreaterThan(tarPatch); + expect(bracePatch.instruction.text, file).toContain( + "--npm-root /usr/local/lib/node_modules/npm", + ); }); }); diff --git a/test/bundled-npm-ip-address-dockerfile-contract.test.ts b/test/bundled-npm-ip-address-dockerfile-contract.test.ts index a5d55ea9281..0284393195a 100644 --- a/test/bundled-npm-ip-address-dockerfile-contract.test.ts +++ b/test/bundled-npm-ip-address-dockerfile-contract.test.ts @@ -12,6 +12,7 @@ import { REVIEWED_NPM_VERSION, } from "../scripts/lib/patch-bundled-npm-ip-address.mts"; import { REVIEWED_NPM_VERSION as UPGRADED_NPM_VERSION } from "../scripts/upgrade-bundled-npm.mts"; +import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; const repoRoot = path.resolve(import.meta.dirname, ".."); const baseDockerfiles = [ @@ -29,10 +30,6 @@ const copyInstruction = const patchCommand = "node --experimental-strip-types /scripts/lib/patch-bundled-npm-ip-address.mts"; -function instructionBody(source: string, start: number): string { - return source.slice(start).split(/\n(?=\S)/u, 1)[0] ?? ""; -} - describe("bundled npm ip-address image remediation contract", () => { it("binds the replacement to the reviewed npm and registry artifact", () => { expect(REVIEWED_NPM_VERSION).toBe(UPGRADED_NPM_VERSION); @@ -49,35 +46,36 @@ describe("bundled npm ip-address image remediation contract", () => { it.each(baseDockerfiles)("patches the reviewed npm tree after upgrading it in %s", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const upgrade = source.indexOf( - "RUN node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", - ); - const patch = source.indexOf(patchCommand); + const upgrade = requireSingleDockerfileRunCommand( + source, + "node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", + ).commandStart; + const patch = requireSingleDockerfileRunCommand(source, patchCommand); expect(copy, file).toBeGreaterThanOrEqual(0); expect(upgrade, file).toBeGreaterThan(copy); - expect(patch, file).toBeGreaterThan(upgrade); - expect(instructionBody(source, patch), file).toContain( - "--npm-root /usr/local/lib/node_modules/npm", - ); + expect(patch.commandStart, file).toBeGreaterThan(upgrade); + expect(patch.instruction.text, file).toContain("--npm-root /usr/local/lib/node_modules/npm"); }); it.each(finalDockerfiles)("reasserts the private package fix in the completed %s", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const tarPatch = source.indexOf( + const tarPatch = requireSingleDockerfileRunCommand( + source, "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", - ); - const bracePatch = source.indexOf( + ).commandStart; + const bracePatch = requireSingleDockerfileRunCommand( + source, "node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts", - ); - const ipAddressPatch = source.indexOf(patchCommand); + ).commandStart; + const ipAddressPatch = requireSingleDockerfileRunCommand(source, patchCommand); expect(copy, file).toBeGreaterThanOrEqual(0); expect(tarPatch, file).toBeGreaterThan(copy); expect(bracePatch, file).toBeGreaterThan(tarPatch); - expect(ipAddressPatch, file).toBeGreaterThan(bracePatch); - expect(instructionBody(source, ipAddressPatch), file).toContain( + expect(ipAddressPatch.commandStart, file).toBeGreaterThan(bracePatch); + expect(ipAddressPatch.instruction.text, file).toContain( "--npm-root /usr/local/lib/node_modules/npm", ); }); diff --git a/test/dockerfile-run-commands.test.ts b/test/dockerfile-run-commands.test.ts new file mode 100644 index 00000000000..48beb574e50 --- /dev/null +++ b/test/dockerfile-run-commands.test.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + findDockerfileRunCommands, + requireSingleDockerfileRunCommand, +} from "./helpers/dockerfile-run-commands"; + +const command = "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts"; + +describe("Dockerfile RUN command discovery", () => { + it("ignores command text in comments, strings, and non-RUN instructions", () => { + const source = [ + `# ${command}`, + `LABEL remediation=\"${command}\"`, + `RUN printf '%s\\n' '${command}'`, + `RUN printf '%s\\n' complete # ${command}`, + "", + ].join("\n"); + + expect(findDockerfileRunCommands(source, command)).toEqual([]); + expect(() => requireSingleDockerfileRunCommand(source, command)).toThrow( + "Expected one executing RUN command", + ); + }); + + it("finds a command after a guard in one complete multiline RUN instruction", () => { + const continuation = "\\"; + const source = [ + `RUN if [ -f /corporate-ca.pem ]; then ${continuation}`, + ` export CURL_CA_BUNDLE=/corporate-ca.pem; ${continuation}`, + ` fi; ${continuation}`, + ` ${command} ${continuation}`, + " --npm-root /usr/local/lib/node_modules/npm", + "ENV NEXT=instruction", + "", + ].join("\n"); + + const match = requireSingleDockerfileRunCommand(source, command); + + expect(match.commandStart).toBe(source.indexOf(command)); + expect(match.instruction.text).toContain("export CURL_CA_BUNDLE=/corporate-ca.pem"); + expect(match.instruction.text).toContain("--npm-root /usr/local/lib/node_modules/npm"); + expect(match.instruction.text).not.toContain("ENV NEXT=instruction"); + }); + + it("reports an extra unguarded command instead of selecting one occurrence", () => { + const source = [`RUN if true; then ${command}; fi`, `RUN ${command}`, ""].join("\n"); + + expect(findDockerfileRunCommands(source, command)).toHaveLength(2); + expect(() => requireSingleDockerfileRunCommand(source, command)).toThrow("found 2"); + }); +}); diff --git a/test/helpers/dockerfile-run-commands.ts b/test/helpers/dockerfile-run-commands.ts new file mode 100644 index 00000000000..f44c51d67c8 --- /dev/null +++ b/test/helpers/dockerfile-run-commands.ts @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface DockerfileInstruction { + readonly body: string; + readonly bodyStart: number; + readonly end: number; + readonly keyword: string; + readonly start: number; + readonly text: string; +} + +export interface DockerfileRunCommand { + readonly commandStart: number; + readonly instruction: DockerfileInstruction; +} + +function lineEnd(source: string, start: number): number { + const newline = source.indexOf("\n", start); + return newline === -1 ? source.length : newline + 1; +} + +function continuesInstruction(line: string): boolean { + const content = line.replace(/\r?\n$/u, "").trimEnd(); + let escapeCount = 0; + for (let index = content.length - 1; index >= 0 && content[index] === "\\"; index -= 1) { + escapeCount += 1; + } + return escapeCount % 2 === 1; +} + +export function dockerfileInstructions(source: string): DockerfileInstruction[] { + const instructions: DockerfileInstruction[] = []; + let offset = 0; + + while (offset < source.length) { + const endOfFirstLine = lineEnd(source, offset); + const firstLine = source.slice(offset, endOfFirstLine); + const instructionMatch = firstLine.match(/^[ \t]*([A-Za-z]+)(?:[ \t]+|(?=\r?$))/u); + if (instructionMatch === null) { + offset = endOfFirstLine; + continue; + } + + let end = endOfFirstLine; + let currentLine = firstLine; + while (continuesInstruction(currentLine)) { + if (end >= source.length) { + throw new Error(`Dockerfile ends inside the ${instructionMatch[1]} instruction`); + } + const nextEnd = lineEnd(source, end); + currentLine = source.slice(end, nextEnd); + end = nextEnd; + } + + const bodyStart = offset + instructionMatch[0].length; + instructions.push({ + body: source.slice(bodyStart, end), + bodyStart, + end, + keyword: instructionMatch[1].toUpperCase(), + start: offset, + text: source.slice(offset, end), + }); + offset = end; + } + + return instructions; +} + +function withoutDockerfileContinuations(source: string): string { + return source.replace(/\\\r?\n/gu, (continuation) => " ".repeat(continuation.length)); +} + +function isCommandStart(source: string, start: number): boolean { + let previous = start - 1; + while (previous >= 0 && /\s/u.test(source[previous])) { + previous -= 1; + } + if (previous < 0 || ";&|({\n".includes(source[previous])) { + return true; + } + + const prefix = source.slice(0, start).trimEnd(); + return /(?:^|[;&|({])\s*(?:then|do|else)$/u.test(prefix); +} + +function shellCommandIndexes(source: string, command: string): number[] { + const indexes: number[] = []; + let quote: "'" | '"' | "`" | null = null; + let comment = false; + + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + if (comment) { + if (character === "\n") comment = false; + continue; + } + if (quote !== null) { + if (character === "\\" && quote !== "'") { + index += 1; + } else if (character === quote) { + quote = null; + } + continue; + } + if (character === "'" || character === '"' || character === "`") { + quote = character; + continue; + } + if (character === "\\") { + index += 1; + continue; + } + if (character === "#" && (index === 0 || /[\s;&|(){}]/u.test(source[index - 1]))) { + comment = true; + continue; + } + if (!source.startsWith(command, index) || !isCommandStart(source, index)) continue; + + const next = source[index + command.length]; + if (next !== undefined && !/[\s;&|(){}]/u.test(next)) continue; + indexes.push(index); + index += command.length - 1; + } + + return indexes; +} + +export function findDockerfileRunCommands(source: string, command: string): DockerfileRunCommand[] { + return dockerfileInstructions(source).flatMap((instruction) => { + if (instruction.keyword !== "RUN") return []; + const shellSource = withoutDockerfileContinuations(instruction.body); + return shellCommandIndexes(shellSource, command).map((commandIndex) => ({ + commandStart: instruction.bodyStart + commandIndex, + instruction, + })); + }); +} + +export function requireSingleDockerfileRunCommand( + source: string, + command: string, +): DockerfileRunCommand { + const matches = findDockerfileRunCommands(source, command); + if (matches.length !== 1) { + throw new Error(`Expected one executing RUN command '${command}', found ${matches.length}`); + } + return matches[0]; +} diff --git a/test/hermes-final-image-layout.test.ts b/test/hermes-final-image-layout.test.ts index 48cc7967219..b576836a282 100644 --- a/test/hermes-final-image-layout.test.ts +++ b/test/hermes-final-image-layout.test.ts @@ -6,6 +6,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; import { dockerRunCommandBetween, runDockerShell } from "./helpers/hermes-dockerfile-run"; import { expectManagedBootstrapNativeImageContract } from "./support/managed-bootstrap-image-contract"; @@ -317,10 +318,10 @@ describe("Hermes final image layout", () => { const runtime = indexOfRequired(finalStage, runtimeCopy); const wrapper = indexOfRequired(finalStage, wrapperCopy); const scan = indexOfRequired(finalStage, scanCopy); - const tarPatch = indexOfRequired( + const tarPatch = requireSingleDockerfileRunCommand( finalStage, "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", - ); + ).commandStart; const certifiInstall = indexOfRequired(finalStage, "RUN _hermes_certifi="); const agentChmod = indexOfRequired( finalStage, diff --git a/test/node-tar-dockerfile-contract.test.ts b/test/node-tar-dockerfile-contract.test.ts index a00d870a50d..0a698ac85c5 100644 --- a/test/node-tar-dockerfile-contract.test.ts +++ b/test/node-tar-dockerfile-contract.test.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { NODE_BASES_REQUIRING_BUNDLED_NPM_TAR_PATCH } from "../scripts/patch-bundled-npm-tar.mts"; +import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; const repoRoot = path.resolve(import.meta.dirname, ".."); const dockerfiles = [ @@ -62,7 +63,7 @@ describe("node-tar image remediation contract", () => { ])("installs curl before patching the bundled npm tar in $file", (file) => { const source = completedStage(fs.readFileSync(path.join(repoRoot, file), "utf8")); const curlInstall = source.indexOf("curl="); - const patchRun = source.indexOf(patchCommand); + const patchRun = requireSingleDockerfileRunCommand(source, patchCommand).commandStart; expect(curlInstall, file).toBeGreaterThanOrEqual(0); expect(patchRun, file).toBeGreaterThan(curlInstall); @@ -89,13 +90,14 @@ describe("node-tar image remediation contract", () => { const patchCopy = patchInputStage.indexOf( "COPY scripts/patch-bundled-npm-tar.mts /scripts/patch-bundled-npm-tar.mts", ); - const patchRun = source.indexOf(patchCommand); + const patchRun = requireSingleDockerfileRunCommand(source, patchCommand).commandStart; const scanCopy = scanInputStage.indexOf( "COPY scripts/checks/node-tar-image-scan.mts /scripts/checks/node-tar-image-scan.mts", ); - const scanRun = source.indexOf( + const scanRun = requireSingleDockerfileRunCommand( + source, "node --experimental-strip-types /scripts/checks/node-tar-image-scan.mts", - ); + ).commandStart; const patchInputReady = patchPayloadLayer >= 0 ? patchPayloadLayer : patchCopy; const scanInputReady = scanPayloadLayer >= 0 ? scanPayloadLayer : scanCopy; @@ -148,13 +150,14 @@ describe("reviewed npm image remediation contract", () => { { file: "agents/langchain-deepagents-code/Dockerfile.base", installsWithNpm: false }, ])("upgrades npm before use in $file", ({ file, installsWithNpm }) => { const source = completedStage(fs.readFileSync(path.join(repoRoot, file), "utf8")); - const patchRun = source.indexOf(patchCommand); + const patchRun = requireSingleDockerfileRunCommand(source, patchCommand).commandStart; const upgradeCopy = source.indexOf( "COPY scripts/upgrade-bundled-npm.mts /scripts/upgrade-bundled-npm.mts", ); - const upgradeRun = source.indexOf( - "RUN node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", - ); + const upgradeRun = requireSingleDockerfileRunCommand( + source, + "node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", + ).commandStart; expect(upgradeCopy, file).toBeGreaterThanOrEqual(0); expect(patchRun, file).toBeGreaterThan(upgradeCopy); From 48b8cd91a99f0364155a4b402307bf72032e12fb Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 09:07:26 -0700 Subject: [PATCH 04/19] fix(security): scope Hermes build CA trust Signed-off-by: Carlos Villela --- agents/hermes/Dockerfile | 8 +- .../security/configure-corporate-ca-trust.mdx | 2 + ...race-expansion-dockerfile-contract.test.ts | 25 +++--- ...npm-ip-address-dockerfile-contract.test.ts | 24 +++--- test/corporate-ca-build-tls-anchor.test.ts | 22 +++++ test/dockerfile-run-commands.test.ts | 61 ++++++++++---- test/helpers/dockerfile-run-commands.ts | 80 ++++++++++--------- test/hermes-final-image-layout.test.ts | 6 +- test/node-tar-dockerfile-contract.test.ts | 29 +++++-- 9 files changed, 174 insertions(+), 83 deletions(-) diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 9deece23072..35fa9636382 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -700,8 +700,14 @@ RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-b # Apply messaging agent-install hooks as root so Hermes Python packages can update # /opt/hermes/.venv before the runtime drops to the sandbox user. WORKDIR /opt/hermes +# uv reads SSL_CERT_FILE for registry TLS. Python build helpers can use +# REQUESTS_CA_BUNDLE. Set both only for package installation when the CA exists. # hadolint ignore=DL3059 -RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts --agent hermes --phase agent-install \ +RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \ + export SSL_CERT_FILE=/usr/local/share/nemoclaw/corporate-ca.pem; \ + export REQUESTS_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \ + fi; \ + node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts --agent hermes --phase agent-install \ && if [ "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" ]; then \ node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts \ --agent hermes --phase managed-image-capability-union; \ diff --git a/docs/security/configure-corporate-ca-trust.mdx b/docs/security/configure-corporate-ca-trust.mdx index a55b5e5c30c..8eca4f93ca7 100644 --- a/docs/security/configure-corporate-ca-trust.mdx +++ b/docs/security/configure-corporate-ca-trust.mdx @@ -42,6 +42,8 @@ It sets `NODE_EXTRA_CA_CERTS` before build-time Node.js dependency verification, The Hermes discovery-runtime installer applies the corporate CA before its npm operations. The final Hermes image stage decodes the CA immediately after `FROM ${BASE_IMAGE}` and sets `NODE_EXTRA_CA_CERTS` before later npm operations. The registry-backed npm remediations set `CURL_CA_BUNDLE` before each download only when the decoded certificate file exists. +The Hermes package installer sets `SSL_CERT_FILE` and `REQUESTS_CA_BUNDLE` for its build-time `uv pip install` commands only when the decoded CA exists. +If the file does not exist, uv and Python keep their default trust configuration. diff --git a/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts b/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts index 5aefb4eb99d..d8124e411d7 100644 --- a/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts +++ b/test/bundled-npm-brace-expansion-dockerfile-contract.test.ts @@ -12,7 +12,7 @@ import { REVIEWED_NPM_VERSION, } from "../scripts/patch-bundled-npm-brace-expansion.mts"; import { REVIEWED_NPM_VERSION as UPGRADED_NPM_VERSION } from "../scripts/upgrade-bundled-npm.mts"; -import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; +import { requireSingleReviewedDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; const repoRoot = path.resolve(import.meta.dirname, ".."); const baseDockerfiles = [ @@ -29,6 +29,7 @@ const copyInstruction = "COPY scripts/patch-bundled-npm-brace-expansion.mts /scripts/patch-bundled-npm-brace-expansion.mts"; const patchInstruction = "node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts"; +const npmRootArguments = ["--npm-root", "/usr/local/lib/node_modules/npm"] as const; describe("bundled npm brace-expansion image remediation contract", () => { it("binds the replacement to the reviewed npm and registry artifact", () => { @@ -44,16 +45,20 @@ describe("bundled npm brace-expansion image remediation contract", () => { it.each(baseDockerfiles)("patches the reviewed npm tree after upgrading it in %s", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const upgrade = requireSingleDockerfileRunCommand( + const upgrade = requireSingleReviewedDockerfileRunCommand( source, "node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", + npmRootArguments, ).commandStart; - const patch = requireSingleDockerfileRunCommand(source, patchInstruction); + const patch = requireSingleReviewedDockerfileRunCommand( + source, + patchInstruction, + npmRootArguments, + ); expect(copy, file).toBeGreaterThanOrEqual(0); expect(upgrade, file).toBeGreaterThan(copy); expect(patch.commandStart, file).toBeGreaterThan(upgrade); - expect(patch.instruction.text, file).toContain("--npm-root /usr/local/lib/node_modules/npm"); }); it.each( @@ -61,17 +66,19 @@ describe("bundled npm brace-expansion image remediation contract", () => { )("reasserts the private package fix in the completed %s filesystem", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const tarPatch = requireSingleDockerfileRunCommand( + const tarPatch = requireSingleReviewedDockerfileRunCommand( source, "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", + npmRootArguments, ).commandStart; - const bracePatch = requireSingleDockerfileRunCommand(source, patchInstruction); + const bracePatch = requireSingleReviewedDockerfileRunCommand( + source, + patchInstruction, + npmRootArguments, + ); expect(copy, file).toBeGreaterThanOrEqual(0); expect(tarPatch, file).toBeGreaterThan(copy); expect(bracePatch.commandStart, file).toBeGreaterThan(tarPatch); - expect(bracePatch.instruction.text, file).toContain( - "--npm-root /usr/local/lib/node_modules/npm", - ); }); }); diff --git a/test/bundled-npm-ip-address-dockerfile-contract.test.ts b/test/bundled-npm-ip-address-dockerfile-contract.test.ts index 0284393195a..01d17988562 100644 --- a/test/bundled-npm-ip-address-dockerfile-contract.test.ts +++ b/test/bundled-npm-ip-address-dockerfile-contract.test.ts @@ -12,7 +12,7 @@ import { REVIEWED_NPM_VERSION, } from "../scripts/lib/patch-bundled-npm-ip-address.mts"; import { REVIEWED_NPM_VERSION as UPGRADED_NPM_VERSION } from "../scripts/upgrade-bundled-npm.mts"; -import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; +import { requireSingleReviewedDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; const repoRoot = path.resolve(import.meta.dirname, ".."); const baseDockerfiles = [ @@ -29,6 +29,7 @@ const copyInstruction = "COPY scripts/lib/patch-bundled-npm-ip-address.mts /scripts/lib/patch-bundled-npm-ip-address.mts"; const patchCommand = "node --experimental-strip-types /scripts/lib/patch-bundled-npm-ip-address.mts"; +const npmRootArguments = ["--npm-root", "/usr/local/lib/node_modules/npm"] as const; describe("bundled npm ip-address image remediation contract", () => { it("binds the replacement to the reviewed npm and registry artifact", () => { @@ -46,37 +47,40 @@ describe("bundled npm ip-address image remediation contract", () => { it.each(baseDockerfiles)("patches the reviewed npm tree after upgrading it in %s", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const upgrade = requireSingleDockerfileRunCommand( + const upgrade = requireSingleReviewedDockerfileRunCommand( source, "node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", + npmRootArguments, ).commandStart; - const patch = requireSingleDockerfileRunCommand(source, patchCommand); + const patch = requireSingleReviewedDockerfileRunCommand(source, patchCommand, npmRootArguments); expect(copy, file).toBeGreaterThanOrEqual(0); expect(upgrade, file).toBeGreaterThan(copy); expect(patch.commandStart, file).toBeGreaterThan(upgrade); - expect(patch.instruction.text, file).toContain("--npm-root /usr/local/lib/node_modules/npm"); }); it.each(finalDockerfiles)("reasserts the private package fix in the completed %s", (file) => { const source = fs.readFileSync(path.join(repoRoot, file), "utf8"); const copy = source.indexOf(copyInstruction); - const tarPatch = requireSingleDockerfileRunCommand( + const tarPatch = requireSingleReviewedDockerfileRunCommand( source, "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", + npmRootArguments, ).commandStart; - const bracePatch = requireSingleDockerfileRunCommand( + const bracePatch = requireSingleReviewedDockerfileRunCommand( source, "node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts", + npmRootArguments, ).commandStart; - const ipAddressPatch = requireSingleDockerfileRunCommand(source, patchCommand); + const ipAddressPatch = requireSingleReviewedDockerfileRunCommand( + source, + patchCommand, + npmRootArguments, + ); expect(copy, file).toBeGreaterThanOrEqual(0); expect(tarPatch, file).toBeGreaterThan(copy); expect(bracePatch, file).toBeGreaterThan(tarPatch); expect(ipAddressPatch.commandStart, file).toBeGreaterThan(bracePatch); - expect(ipAddressPatch.instruction.text, file).toContain( - "--npm-root /usr/local/lib/node_modules/npm", - ); }); }); diff --git a/test/corporate-ca-build-tls-anchor.test.ts b/test/corporate-ca-build-tls-anchor.test.ts index 20c4982be76..f30644ee17e 100644 --- a/test/corporate-ca-build-tls-anchor.test.ts +++ b/test/corporate-ca-build-tls-anchor.test.ts @@ -4,6 +4,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { dockerfileInstructions } from "./helpers/dockerfile-run-commands"; const DOCKERFILE = join(import.meta.dirname, "../Dockerfile"); @@ -205,6 +206,24 @@ describe("Hermes corporate proxy CA final-stage trust", () => { "node --experimental-strip-types /scripts/patch-bundled-npm-brace-expansion.mts", "node --experimental-strip-types /scripts/lib/patch-bundled-npm-ip-address.mts", ]; + const agentInstallCommand = + "node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts --agent hermes --phase agent-install"; + const packageInstallRun = dockerfileInstructions(finalStage).find( + (instruction) => + instruction.keyword === "RUN" && instruction.body.includes(agentInstallCommand), + ); + const expectedPackageInstallRun = [ + "RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \\", + " export SSL_CERT_FILE=/usr/local/share/nemoclaw/corporate-ca.pem; \\", + " export REQUESTS_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \\", + " fi; \\", + ` ${agentInstallCommand} \\`, + ' && if [ "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" ]; then \\', + " node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts \\", + " --agent hermes --phase managed-image-capability-union; \\", + " fi", + "", + ].join("\n"); const npmCommandIndexes = [...finalStage.matchAll(/^\s*npm\s+(?:ci|run)\b/gmu)].map( (match) => match.index, ); @@ -234,5 +253,8 @@ describe("Hermes corporate proxy CA final-stage trust", () => { for (const npmCommandIndex of npmCommandIndexes) { expect(nodeAnchorIndex).toBeLessThan(npmCommandIndex); } + expect(packageInstallRun?.text).toBe(expectedPackageInstallRun); + expect(packageInstallRun?.text).not.toContain("else"); + expect(finalStage.match(/^ENV (?:SSL_CERT_FILE|REQUESTS_CA_BUNDLE)=/gmu) ?? []).toEqual([]); }); }); diff --git a/test/dockerfile-run-commands.test.ts b/test/dockerfile-run-commands.test.ts index 48beb574e50..a3823c2fb9e 100644 --- a/test/dockerfile-run-commands.test.ts +++ b/test/dockerfile-run-commands.test.ts @@ -2,12 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import { - findDockerfileRunCommands, - requireSingleDockerfileRunCommand, -} from "./helpers/dockerfile-run-commands"; +import { requireSingleReviewedDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; const command = "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts"; +const corporateCaPath = "/usr/local/share/nemoclaw/corporate-ca.pem"; +const requiredArguments = ["--npm-root", "/usr/local/lib/node_modules/npm"] as const; +const invocation = [command, ...requiredArguments].join(" "); describe("Dockerfile RUN command discovery", () => { it("ignores command text in comments, strings, and non-RUN instructions", () => { @@ -19,36 +19,65 @@ describe("Dockerfile RUN command discovery", () => { "", ].join("\n"); - expect(findDockerfileRunCommands(source, command)).toEqual([]); - expect(() => requireSingleDockerfileRunCommand(source, command)).toThrow( - "Expected one executing RUN command", - ); + expect(() => + requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments), + ).toThrow("Expected one reviewed RUN command"); + }); + + it("accepts the reviewed command and arguments as a direct RUN instruction", () => { + const source = `RUN ${invocation}\nENV NEXT=instruction\n`; + + const match = requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments); + + expect(match.commandStart).toBe(source.indexOf(command)); + expect(match.instruction.text).toBe(`RUN ${invocation}\n`); }); it("finds a command after a guard in one complete multiline RUN instruction", () => { const continuation = "\\"; const source = [ - `RUN if [ -f /corporate-ca.pem ]; then ${continuation}`, - ` export CURL_CA_BUNDLE=/corporate-ca.pem; ${continuation}`, + `RUN if [ -f ${corporateCaPath} ]; then ${continuation}`, + ` export CURL_CA_BUNDLE=${corporateCaPath}; ${continuation}`, ` fi; ${continuation}`, ` ${command} ${continuation}`, - " --npm-root /usr/local/lib/node_modules/npm", + ` ${requiredArguments.join(" ")}`, "ENV NEXT=instruction", "", ].join("\n"); - const match = requireSingleDockerfileRunCommand(source, command); + const match = requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments); expect(match.commandStart).toBe(source.indexOf(command)); - expect(match.instruction.text).toContain("export CURL_CA_BUNDLE=/corporate-ca.pem"); + expect(match.instruction.text).toContain(`export CURL_CA_BUNDLE=${corporateCaPath}`); expect(match.instruction.text).toContain("--npm-root /usr/local/lib/node_modules/npm"); expect(match.instruction.text).not.toContain("ENV NEXT=instruction"); }); it("reports an extra unguarded command instead of selecting one occurrence", () => { - const source = [`RUN if true; then ${command}; fi`, `RUN ${command}`, ""].join("\n"); + const source = [`RUN ${invocation}`, `RUN ${invocation}`, ""].join("\n"); + + expect(() => + requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments), + ).toThrow("found 2"); + }); + + it.each([ + ["a short-circuit branch", `RUN false && ${invocation}\n`], + ["an uncalled function", `RUN patch() { ${invocation}; }; true\n`], + ["an unreachable conditional branch", `RUN if false; then ${invocation}; fi\n`], + ])("rejects the reviewed command inside %s", (_label, source) => { + expect(() => + requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments), + ).toThrow("unreviewed RUN instruction"); + }); - expect(findDockerfileRunCommands(source, command)).toHaveLength(2); - expect(() => requireSingleDockerfileRunCommand(source, command)).toThrow("found 2"); + it.each([ + ["before the command", `RUN printf '%s' '${requiredArguments.join(" ")}'; ${command}\n`], + ["in one quoted value", `RUN ${command} '${requiredArguments.join(" ")}'\n`], + ["in a comment", `RUN ${command} # ${requiredArguments.join(" ")}\n`], + ])("rejects required arguments that occur %s", (_label, source) => { + expect(() => + requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments), + ).toThrow("unreviewed RUN instruction"); }); }); diff --git a/test/helpers/dockerfile-run-commands.ts b/test/helpers/dockerfile-run-commands.ts index f44c51d67c8..e1f5b0eef69 100644 --- a/test/helpers/dockerfile-run-commands.ts +++ b/test/helpers/dockerfile-run-commands.ts @@ -10,11 +10,14 @@ export interface DockerfileInstruction { readonly text: string; } -export interface DockerfileRunCommand { +export interface ReviewedDockerfileRunCommand { readonly commandStart: number; readonly instruction: DockerfileInstruction; } +const CORPORATE_CA_PATH = "/usr/local/share/nemoclaw/corporate-ca.pem"; +const CORPORATE_CA_GUARD = `if [ -f ${CORPORATE_CA_PATH} ]; then export CURL_CA_BUNDLE=${CORPORATE_CA_PATH}; fi;`; + function lineEnd(source: string, start: number): number { const newline = source.indexOf("\n", start); return newline === -1 ? source.length : newline + 1; @@ -68,24 +71,7 @@ export function dockerfileInstructions(source: string): DockerfileInstruction[] return instructions; } -function withoutDockerfileContinuations(source: string): string { - return source.replace(/\\\r?\n/gu, (continuation) => " ".repeat(continuation.length)); -} - -function isCommandStart(source: string, start: number): boolean { - let previous = start - 1; - while (previous >= 0 && /\s/u.test(source[previous])) { - previous -= 1; - } - if (previous < 0 || ";&|({\n".includes(source[previous])) { - return true; - } - - const prefix = source.slice(0, start).trimEnd(); - return /(?:^|[;&|({])\s*(?:then|do|else)$/u.test(prefix); -} - -function shellCommandIndexes(source: string, command: string): number[] { +function unquotedTextIndexes(source: string, text: string): number[] { const indexes: number[] = []; let quote: "'" | '"' | "`" | null = null; let comment = false; @@ -116,35 +102,55 @@ function shellCommandIndexes(source: string, command: string): number[] { comment = true; continue; } - if (!source.startsWith(command, index) || !isCommandStart(source, index)) continue; - - const next = source[index + command.length]; - if (next !== undefined && !/[\s;&|(){}]/u.test(next)) continue; + if (!source.startsWith(text, index)) continue; indexes.push(index); - index += command.length - 1; + index += text.length - 1; } return indexes; } -export function findDockerfileRunCommands(source: string, command: string): DockerfileRunCommand[] { - return dockerfileInstructions(source).flatMap((instruction) => { - if (instruction.keyword !== "RUN") return []; - const shellSource = withoutDockerfileContinuations(instruction.body); - return shellCommandIndexes(shellSource, command).map((commandIndex) => ({ - commandStart: instruction.bodyStart + commandIndex, - instruction, - })); - }); +function normalizedInstructionBody(source: string): string { + return source + .replace(/\\\r?\n/gu, " ") + .replace(/\s+/gu, " ") + .trim(); } -export function requireSingleDockerfileRunCommand( +export function requireSingleReviewedDockerfileRunCommand( source: string, command: string, -): DockerfileRunCommand { - const matches = findDockerfileRunCommands(source, command); + requiredArguments: readonly string[], +): ReviewedDockerfileRunCommand { + const invocation = [command, ...requiredArguments].join(" "); + const reviewedBodies = new Set([invocation, `${CORPORATE_CA_GUARD} ${invocation}`]); + const matches: ReviewedDockerfileRunCommand[] = []; + let unreviewedInstructions = 0; + + for (const instruction of dockerfileInstructions(source)) { + if (instruction.keyword !== "RUN") continue; + const commandIndexes = unquotedTextIndexes(instruction.body, command); + if (commandIndexes.length === 0) continue; + if ( + commandIndexes.length !== 1 || + !reviewedBodies.has(normalizedInstructionBody(instruction.body)) + ) { + unreviewedInstructions += 1; + continue; + } + matches.push({ + commandStart: instruction.bodyStart + commandIndexes[0], + instruction, + }); + } + + if (unreviewedInstructions > 0) { + throw new Error( + `Expected '${invocation}' only as a direct RUN or the reviewed corporate CA guarded RUN; found ${unreviewedInstructions} unreviewed RUN instruction(s)`, + ); + } if (matches.length !== 1) { - throw new Error(`Expected one executing RUN command '${command}', found ${matches.length}`); + throw new Error(`Expected one reviewed RUN command '${invocation}', found ${matches.length}`); } return matches[0]; } diff --git a/test/hermes-final-image-layout.test.ts b/test/hermes-final-image-layout.test.ts index b576836a282..0ed2c5aa3bf 100644 --- a/test/hermes-final-image-layout.test.ts +++ b/test/hermes-final-image-layout.test.ts @@ -6,12 +6,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; +import { requireSingleReviewedDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; import { dockerRunCommandBetween, runDockerShell } from "./helpers/hermes-dockerfile-run"; import { expectManagedBootstrapNativeImageContract } from "./support/managed-bootstrap-image-contract"; const ROOT = path.resolve(import.meta.dirname, ".."); const HERMES_DOCKERFILE = path.join(ROOT, "agents", "hermes", "Dockerfile"); +const NPM_ROOT_ARGUMENTS = ["--npm-root", "/usr/local/lib/node_modules/npm"] as const; const HERMES_INTEGRITY_FILES = [ { arg: "NEMOCLAW_HERMES_IMAGE_BUILD_PROBES_SHA256", @@ -318,9 +319,10 @@ describe("Hermes final image layout", () => { const runtime = indexOfRequired(finalStage, runtimeCopy); const wrapper = indexOfRequired(finalStage, wrapperCopy); const scan = indexOfRequired(finalStage, scanCopy); - const tarPatch = requireSingleDockerfileRunCommand( + const tarPatch = requireSingleReviewedDockerfileRunCommand( finalStage, "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts", + NPM_ROOT_ARGUMENTS, ).commandStart; const certifiInstall = indexOfRequired(finalStage, "RUN _hermes_certifi="); const agentChmod = indexOfRequired( diff --git a/test/node-tar-dockerfile-contract.test.ts b/test/node-tar-dockerfile-contract.test.ts index 0a698ac85c5..0e4eca19c38 100644 --- a/test/node-tar-dockerfile-contract.test.ts +++ b/test/node-tar-dockerfile-contract.test.ts @@ -7,7 +7,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { NODE_BASES_REQUIRING_BUNDLED_NPM_TAR_PATCH } from "../scripts/patch-bundled-npm-tar.mts"; -import { requireSingleDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; +import { requireSingleReviewedDockerfileRunCommand } from "./helpers/dockerfile-run-commands"; const repoRoot = path.resolve(import.meta.dirname, ".."); const dockerfiles = [ @@ -31,6 +31,7 @@ const dockerfiles = [ }, ] as const; const patchCommand = "node --experimental-strip-types /scripts/patch-bundled-npm-tar.mts"; +const npmRootArguments = ["--npm-root", "/usr/local/lib/node_modules/npm"] as const; function completedStage(source: string): string { const finalStageStart = [...source.matchAll(/^FROM\b/gmu)].at(-1)?.index; @@ -63,7 +64,11 @@ describe("node-tar image remediation contract", () => { ])("installs curl before patching the bundled npm tar in $file", (file) => { const source = completedStage(fs.readFileSync(path.join(repoRoot, file), "utf8")); const curlInstall = source.indexOf("curl="); - const patchRun = requireSingleDockerfileRunCommand(source, patchCommand).commandStart; + const patchRun = requireSingleReviewedDockerfileRunCommand( + source, + patchCommand, + npmRootArguments, + ).commandStart; expect(curlInstall, file).toBeGreaterThanOrEqual(0); expect(patchRun, file).toBeGreaterThan(curlInstall); @@ -90,14 +95,17 @@ describe("node-tar image remediation contract", () => { const patchCopy = patchInputStage.indexOf( "COPY scripts/patch-bundled-npm-tar.mts /scripts/patch-bundled-npm-tar.mts", ); - const patchRun = requireSingleDockerfileRunCommand(source, patchCommand).commandStart; + const patchRun = requireSingleReviewedDockerfileRunCommand( + source, + patchCommand, + npmRootArguments, + ).commandStart; const scanCopy = scanInputStage.indexOf( "COPY scripts/checks/node-tar-image-scan.mts /scripts/checks/node-tar-image-scan.mts", ); - const scanRun = requireSingleDockerfileRunCommand( - source, + const scanRun = source.indexOf( "node --experimental-strip-types /scripts/checks/node-tar-image-scan.mts", - ).commandStart; + ); const patchInputReady = patchPayloadLayer >= 0 ? patchPayloadLayer : patchCopy; const scanInputReady = scanPayloadLayer >= 0 ? scanPayloadLayer : scanCopy; @@ -150,13 +158,18 @@ describe("reviewed npm image remediation contract", () => { { file: "agents/langchain-deepagents-code/Dockerfile.base", installsWithNpm: false }, ])("upgrades npm before use in $file", ({ file, installsWithNpm }) => { const source = completedStage(fs.readFileSync(path.join(repoRoot, file), "utf8")); - const patchRun = requireSingleDockerfileRunCommand(source, patchCommand).commandStart; + const patchRun = requireSingleReviewedDockerfileRunCommand( + source, + patchCommand, + npmRootArguments, + ).commandStart; const upgradeCopy = source.indexOf( "COPY scripts/upgrade-bundled-npm.mts /scripts/upgrade-bundled-npm.mts", ); - const upgradeRun = requireSingleDockerfileRunCommand( + const upgradeRun = requireSingleReviewedDockerfileRunCommand( source, "node --experimental-strip-types /scripts/upgrade-bundled-npm.mts", + npmRootArguments, ).commandStart; expect(upgradeCopy, file).toBeGreaterThanOrEqual(0); From f44498c79a22585c2fec9ba510ee59445a6f92c0 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 09:21:55 -0700 Subject: [PATCH 05/19] fix(security): preserve Hermes package trust defaults Signed-off-by: Carlos Villela --- agents/hermes/Dockerfile | 7 ++++--- docs/security/configure-corporate-ca-trust.mdx | 5 +++-- test/corporate-ca-build-tls-anchor.test.ts | 3 ++- test/dockerfile-run-commands.test.ts | 9 +++++++++ test/helpers/dockerfile-run-commands.ts | 4 ++-- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 35fa9636382..438e8a4b774 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -700,10 +700,11 @@ RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-b # Apply messaging agent-install hooks as root so Hermes Python packages can update # /opt/hermes/.venv before the runtime drops to the sandbox user. WORKDIR /opt/hermes -# uv reads SSL_CERT_FILE for registry TLS. Python build helpers can use -# REQUESTS_CA_BUNDLE. Set both only for package installation when the CA exists. +# Clear inherited Python and uv trust overrides before package installation. +# When the decoded corporate CA exists, use it only for this RUN instruction. # hadolint ignore=DL3059 -RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \ +RUN unset SSL_CERT_FILE REQUESTS_CA_BUNDLE; \ + if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \ export SSL_CERT_FILE=/usr/local/share/nemoclaw/corporate-ca.pem; \ export REQUESTS_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \ fi; \ diff --git a/docs/security/configure-corporate-ca-trust.mdx b/docs/security/configure-corporate-ca-trust.mdx index 8eca4f93ca7..7b85bd677fe 100644 --- a/docs/security/configure-corporate-ca-trust.mdx +++ b/docs/security/configure-corporate-ca-trust.mdx @@ -42,8 +42,9 @@ It sets `NODE_EXTRA_CA_CERTS` before build-time Node.js dependency verification, The Hermes discovery-runtime installer applies the corporate CA before its npm operations. The final Hermes image stage decodes the CA immediately after `FROM ${BASE_IMAGE}` and sets `NODE_EXTRA_CA_CERTS` before later npm operations. The registry-backed npm remediations set `CURL_CA_BUNDLE` before each download only when the decoded certificate file exists. -The Hermes package installer sets `SSL_CERT_FILE` and `REQUESTS_CA_BUNDLE` for its build-time `uv pip install` commands only when the decoded CA exists. -If the file does not exist, uv and Python keep their default trust configuration. +The Hermes package installer clears inherited `SSL_CERT_FILE` and `REQUESTS_CA_BUNDLE` values before its build-time `uv pip install` commands. +When the decoded CA exists, it sets both variables to that file for those commands. +If the file does not exist, uv and Python use their default trust configuration. diff --git a/test/corporate-ca-build-tls-anchor.test.ts b/test/corporate-ca-build-tls-anchor.test.ts index f30644ee17e..cb168608505 100644 --- a/test/corporate-ca-build-tls-anchor.test.ts +++ b/test/corporate-ca-build-tls-anchor.test.ts @@ -213,7 +213,8 @@ describe("Hermes corporate proxy CA final-stage trust", () => { instruction.keyword === "RUN" && instruction.body.includes(agentInstallCommand), ); const expectedPackageInstallRun = [ - "RUN if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \\", + "RUN unset SSL_CERT_FILE REQUESTS_CA_BUNDLE; \\", + " if [ -f /usr/local/share/nemoclaw/corporate-ca.pem ]; then \\", " export SSL_CERT_FILE=/usr/local/share/nemoclaw/corporate-ca.pem; \\", " export REQUESTS_CA_BUNDLE=/usr/local/share/nemoclaw/corporate-ca.pem; \\", " fi; \\", diff --git a/test/dockerfile-run-commands.test.ts b/test/dockerfile-run-commands.test.ts index a3823c2fb9e..ca03c4d187c 100644 --- a/test/dockerfile-run-commands.test.ts +++ b/test/dockerfile-run-commands.test.ts @@ -80,4 +80,13 @@ describe("Dockerfile RUN command discovery", () => { requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments), ).toThrow("unreviewed RUN instruction"); }); + + it.each([ + ["between the command and arguments", `RUN ${command}\u00a0${requiredArguments.join(" ")}\n`], + ["after the arguments", `RUN ${invocation}\u00a0\n`], + ])("rejects non-shell whitespace %s", (_label, source) => { + expect(() => + requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments), + ).toThrow("unreviewed RUN instruction"); + }); }); diff --git a/test/helpers/dockerfile-run-commands.ts b/test/helpers/dockerfile-run-commands.ts index e1f5b0eef69..b96f4065817 100644 --- a/test/helpers/dockerfile-run-commands.ts +++ b/test/helpers/dockerfile-run-commands.ts @@ -113,8 +113,8 @@ function unquotedTextIndexes(source: string, text: string): number[] { function normalizedInstructionBody(source: string): string { return source .replace(/\\\r?\n/gu, " ") - .replace(/\s+/gu, " ") - .trim(); + .replace(/[ \t\r\n]+/gu, " ") + .replace(/^[ \t\r\n]+|[ \t\r\n]+$/gu, ""); } export function requireSingleReviewedDockerfileRunCommand( From d398aca3922379db8d9f33bf778fc662ec219531 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 09:54:43 -0700 Subject: [PATCH 06/19] fix(security): reject hidden Dockerfile commands Signed-off-by: Carlos Villela --- test/dockerfile-run-commands.test.ts | 12 ++++++++++++ test/helpers/dockerfile-run-commands.ts | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/test/dockerfile-run-commands.test.ts b/test/dockerfile-run-commands.test.ts index ca03c4d187c..77f1709d671 100644 --- a/test/dockerfile-run-commands.test.ts +++ b/test/dockerfile-run-commands.test.ts @@ -61,6 +61,18 @@ describe("Dockerfile RUN command discovery", () => { ).toThrow("found 2"); }); + it.each([ + ["a command substitution", `RUN printf '%s\\n' "$(${invocation})"\n`], + ["backticks", `RUN printf '%s\\n' \`${invocation}\`\n`], + ["a parameter expansion", `RUN : \${#PATH}; ${invocation}\n`], + ])("rejects an extra invocation hidden by %s", (_label, hiddenInvocation) => { + const source = [`RUN ${invocation}`, hiddenInvocation].join("\n"); + + expect(() => + requireSingleReviewedDockerfileRunCommand(source, command, requiredArguments), + ).toThrow("unreviewed RUN instruction"); + }); + it.each([ ["a short-circuit branch", `RUN false && ${invocation}\n`], ["an uncalled function", `RUN patch() { ${invocation}; }; true\n`], diff --git a/test/helpers/dockerfile-run-commands.ts b/test/helpers/dockerfile-run-commands.ts index b96f4065817..553e8b38093 100644 --- a/test/helpers/dockerfile-run-commands.ts +++ b/test/helpers/dockerfile-run-commands.ts @@ -129,6 +129,14 @@ export function requireSingleReviewedDockerfileRunCommand( for (const instruction of dockerfileInstructions(source)) { if (instruction.keyword !== "RUN") continue; + const containsCommand = instruction.body.includes(command); + const hasUnsupportedShellConstruct = ["$(", "${", "`"].some((token) => + instruction.body.includes(token), + ); + if (containsCommand && hasUnsupportedShellConstruct) { + unreviewedInstructions += 1; + continue; + } const commandIndexes = unquotedTextIndexes(instruction.body, command); if (commandIndexes.length === 0) continue; if ( From 3c776ce503b5c187486f7b163f1e675a0f1b9841 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 10:04:31 -0700 Subject: [PATCH 07/19] fix(security): scan Dockerfile continuations safely Signed-off-by: Carlos Villela --- test/dockerfile-run-commands.test.ts | 12 ++++++---- test/helpers/dockerfile-run-commands.ts | 32 +++++++++++++++++++++---- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/test/dockerfile-run-commands.test.ts b/test/dockerfile-run-commands.test.ts index 77f1709d671..1aa4f3eb85b 100644 --- a/test/dockerfile-run-commands.test.ts +++ b/test/dockerfile-run-commands.test.ts @@ -8,6 +8,7 @@ const command = "node --experimental-strip-types /scripts/patch-bundled-npm-tar. const corporateCaPath = "/usr/local/share/nemoclaw/corporate-ca.pem"; const requiredArguments = ["--npm-root", "/usr/local/lib/node_modules/npm"] as const; const invocation = [command, ...requiredArguments].join(" "); +const splicedCommand = command.replace("strip-types", "strip-\\\ntypes"); describe("Dockerfile RUN command discovery", () => { it("ignores command text in comments, strings, and non-RUN instructions", () => { @@ -62,10 +63,13 @@ describe("Dockerfile RUN command discovery", () => { }); it.each([ - ["a command substitution", `RUN printf '%s\\n' "$(${invocation})"\n`], - ["backticks", `RUN printf '%s\\n' \`${invocation}\`\n`], - ["a parameter expansion", `RUN : \${#PATH}; ${invocation}\n`], - ])("rejects an extra invocation hidden by %s", (_label, hiddenInvocation) => { + ["inside a command substitution", `RUN printf '%s\\n' "$(${invocation})"\n`], + ["inside backticks", `RUN printf '%s\\n' \`${invocation}\`\n`], + ["after a parameter-length expansion", `RUN : \${#PATH}; ${invocation}\n`], + ["inside a split command substitution", `RUN printf '%s\\n' "$\\\n(${invocation})"\n`], + ["after a split parameter-length expansion", `RUN : $\\\r\n{#PATH}; ${invocation}\n`], + ["with a spliced command token", `RUN ${splicedCommand} ${requiredArguments.join(" ")}\n`], + ])("rejects an extra invocation %s", (_label, hiddenInvocation) => { const source = [`RUN ${invocation}`, hiddenInvocation].join("\n"); expect(() => diff --git a/test/helpers/dockerfile-run-commands.ts b/test/helpers/dockerfile-run-commands.ts index 553e8b38093..723d505a633 100644 --- a/test/helpers/dockerfile-run-commands.ts +++ b/test/helpers/dockerfile-run-commands.ts @@ -71,6 +71,29 @@ export function dockerfileInstructions(source: string): DockerfileInstruction[] return instructions; } +function collapseDockerfileContinuations(source: string): { + readonly originalIndexes: readonly number[]; + readonly text: string; +} { + const characters: string[] = []; + const originalIndexes: number[] = []; + + for (let index = 0; index < source.length; index += 1) { + if (source[index] === "\\" && source[index + 1] === "\n") { + index += 1; + continue; + } + if (source[index] === "\\" && source[index + 1] === "\r" && source[index + 2] === "\n") { + index += 2; + continue; + } + characters.push(source[index]); + originalIndexes.push(index); + } + + return { originalIndexes, text: characters.join("") }; +} + function unquotedTextIndexes(source: string, text: string): number[] { const indexes: number[] = []; let quote: "'" | '"' | "`" | null = null; @@ -129,15 +152,16 @@ export function requireSingleReviewedDockerfileRunCommand( for (const instruction of dockerfileInstructions(source)) { if (instruction.keyword !== "RUN") continue; - const containsCommand = instruction.body.includes(command); + const collapsed = collapseDockerfileContinuations(instruction.body); + const containsCommand = collapsed.text.includes(command); const hasUnsupportedShellConstruct = ["$(", "${", "`"].some((token) => - instruction.body.includes(token), + collapsed.text.includes(token), ); if (containsCommand && hasUnsupportedShellConstruct) { unreviewedInstructions += 1; continue; } - const commandIndexes = unquotedTextIndexes(instruction.body, command); + const commandIndexes = unquotedTextIndexes(collapsed.text, command); if (commandIndexes.length === 0) continue; if ( commandIndexes.length !== 1 || @@ -147,7 +171,7 @@ export function requireSingleReviewedDockerfileRunCommand( continue; } matches.push({ - commandStart: instruction.bodyStart + commandIndexes[0], + commandStart: instruction.bodyStart + collapsed.originalIndexes[commandIndexes[0]], instruction, }); } From 4c9e97cac02dd925f2b30e21257909ff2462db5a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 14:08:35 -0400 Subject: [PATCH 08/19] test(hermes): exercise corporate CA image build Signed-off-by: Julie Yaunches --- .github/workflows/sandbox-images-and-e2e.yaml | 8 +++++++ .../sandbox-images-workflow-boundary.test.ts | 21 +++++++++++++++---- .../e2e/sandbox-images-workflow-boundary.mts | 7 +++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index c74e8a0a2b1..74a68199cf3 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -413,14 +413,22 @@ jobs: shell: bash run: | set -euo pipefail + # curl and Python replace their default roots with this build argument. + corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt + test -s "$corporate_ca_bundle" + corporate_ca_b64="$(base64 -w 0 "$corporate_ca_bundle")" messaging_plan_b64="$(node --experimental-strip-types scripts/check-messaging-plan-image-boundary.mts plan hermes)" build_args=( -f agents/hermes/Dockerfile --build-arg "BASE_IMAGE=${HERMES_BASE_IMAGE}" + --build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}" --build-arg "NEMOCLAW_MESSAGING_PLAN_B64=${messaging_plan_b64}" ) scripts/check-production-build-args.sh "${build_args[@]}" docker build "${build_args[@]}" -t nemoclaw-hermes-plan-boundary . + docker run --rm --network none --entrypoint openssl \ + nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl \ + -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null node --experimental-strip-types scripts/check-messaging-plan-image-boundary.mts verify \ nemoclaw-hermes-plan-boundary hermes diff --git a/test/e2e/support/sandbox-images-workflow-boundary.test.ts b/test/e2e/support/sandbox-images-workflow-boundary.test.ts index defcf647deb..27c74bc7247 100644 --- a/test/e2e/support/sandbox-images-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-images-workflow-boundary.test.ts @@ -428,10 +428,20 @@ describe("sandbox image workflow boundary", () => { const hermes = probe.steps!.find( (step) => step.name === "Build and verify Hermes messaging plan boundary", )!; - hermes.run = hermes.run!.replace( - "check-messaging-plan-image-boundary.mts verify", - "check-messaging-plan-image-boundary.mts bypass", - ); + hermes.run = hermes + .run!.replace( + "corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt", + "corporate_ca_bundle=/missing/ca-certificates.crt", + ) + .replace( + '--build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"', + '--build-arg "NEMOCLAW_CORPORATE_CA_B64="', + ) + .replace("crl2pkcs7 -nocrl", "version") + .replace( + "check-messaging-plan-image-boundary.mts verify", + "check-messaging-plan-image-boundary.mts bypass", + ); expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( expect.arrayContaining([ @@ -440,6 +450,9 @@ describe("sandbox image workflow boundary", () => { "messaging plan image boundary must set up Node exactly once", "messaging plan image boundary must use Node 22.19.0", 'openclaw messaging plan image boundary must include scripts/check-production-build-args.sh "${build_args[@]}"', + 'hermes messaging plan image boundary must include corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt test -s "$corporate_ca_bundle" corporate_ca_b64="$(base64 -w 0 "$corporate_ca_bundle")"', + 'hermes messaging plan image boundary must include --build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"', + "hermes messaging plan image boundary must include docker run --rm --network none --entrypoint openssl nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null", "hermes messaging plan image boundary must include node --experimental-strip-types scripts/check-messaging-plan-image-boundary.mts verify nemoclaw-hermes-plan-boundary hermes", "messaging plan image boundary must not publish probe image artifacts", ]), diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index 83830173735..b9dd00a7165 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -597,6 +597,7 @@ function validateMessagingPlanBoundaryBuild( readonly agent: "hermes" | "openclaw"; readonly baseArgName: "BASE_IMAGE"; readonly baseEnvName: "BASE_IMAGE" | "HERMES_BASE_IMAGE"; + readonly extraRequiredFragments?: readonly string[]; readonly stepName: string; readonly target: string; }, @@ -614,6 +615,7 @@ function validateMessagingPlanBoundaryBuild( 'scripts/check-production-build-args.sh "${build_args[@]}"', `docker build \"\${build_args[@]}\" -t ${options.target} .`, `node --experimental-strip-types scripts/check-messaging-plan-image-boundary.mts verify ${options.target} ${options.agent}`, + ...(options.extraRequiredFragments ?? []), ]; if (step.shell !== "bash" || !isDeepStrictEqual(record(step.env), expectedEnv)) { @@ -677,6 +679,11 @@ function validateMessagingPlanImageBoundary( agent: "hermes", baseArgName: "BASE_IMAGE", baseEnvName: "HERMES_BASE_IMAGE", + extraRequiredFragments: [ + 'corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt test -s "$corporate_ca_bundle" corporate_ca_b64="$(base64 -w 0 "$corporate_ca_bundle")"', + '--build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"', + "docker run --rm --network none --entrypoint openssl nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null", + ], stepName: "Build and verify Hermes messaging plan boundary", target: "nemoclaw-hermes-plan-boundary", }); From ff6421702b7a142f2016335c7df632049b47e6ac Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 14:51:00 -0400 Subject: [PATCH 09/19] test(hermes): verify default CA trust in image Signed-off-by: Julie Yaunches --- .github/workflows/sandbox-images-and-e2e.yaml | 24 ++++++++++++++++++ .../sandbox-images-workflow-boundary.test.ts | 21 ++++++++++++++++ .../e2e/sandbox-images-workflow-boundary.mts | 25 +++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 74a68199cf3..1a1d1793fd1 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -218,6 +218,30 @@ jobs: cache-from: type=gha,scope=hermes-production-${{ runner.os }}-${{ runner.arch }} cache-to: type=gha,mode=max,scope=hermes-production-${{ runner.os }}-${{ runner.arch }} + # The production build intentionally omits NEMOCLAW_CORPORATE_CA_B64. A + # successful final stage therefore proves its registry remediations and + # Hermes agent-install phase complete with the base image's default trust. + - name: Verify Hermes default-trust final image + shell: bash + run: | + set -euo pipefail + docker run --rm \ + --network none \ + --read-only \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --pids-limit 64 \ + --memory 256m \ + --entrypoint /bin/sh \ + nemoclaw-hermes-production -eu -c ' + test "$NODE_EXTRA_CA_CERTS" = /usr/local/share/nemoclaw/corporate-ca.pem + test ! -e /usr/local/share/nemoclaw/corporate-ca.pem + test ! -L /usr/local/share/nemoclaw/corporate-ca.pem + test -x /usr/local/bin/hermes + node -e "const tls = require(\"node:tls\"); if (tls.rootCertificates.length === 0) process.exit(1); tls.createSecureContext()" + /opt/hermes/.venv/bin/python -I -c "import ssl; assert ssl.create_default_context().get_ca_certs()" + ' + - name: Scan completed Hermes image for node-tar id: node-tar-scan shell: bash diff --git a/test/e2e/support/sandbox-images-workflow-boundary.test.ts b/test/e2e/support/sandbox-images-workflow-boundary.test.ts index 27c74bc7247..7da4e1e6586 100644 --- a/test/e2e/support/sandbox-images-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-images-workflow-boundary.test.ts @@ -153,6 +153,27 @@ describe("sandbox image workflow boundary", () => { ); }); + it("requires the canonical no-CA Hermes build and its default-trust image proof", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const producer = imageWorkflow.jobs["build-hermes-sandbox-image"]; + const build = producer.steps!.find((step) => step.name === "Build Hermes production image")!; + build.with!["build-args"] = `${build.with!["build-args"]}\nNEMOCLAW_CORPORATE_CA_B64=test`; + const proof = producer.steps!.find( + (step) => step.name === "Verify Hermes default-trust final image", + )!; + proof.run = proof.run!.replace( + "test ! -e /usr/local/share/nemoclaw/corporate-ca.pem", + "test -e /usr/local/share/nemoclaw/corporate-ca.pem", + ); + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( + expect.arrayContaining([ + "Hermes producer must build the production image exactly once with the canonical local-load Buildx action and OS/architecture-scoped GHA cache", + "Hermes producer must prove the no-CA final image uses default trust before completed-image scans", + ]), + ); + }); + it("rejects non-canonical Hermes Buildx action pins", () => { for (const stepName of ["Set up Docker Buildx", "Build Hermes production image"]) { const { imageWorkflow, mainWorkflow } = readWorkflows(); diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index b9dd00a7165..5214e080699 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -27,6 +27,7 @@ const HERMES_SETUP_BUILDX_ACTION = "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c"; const HERMES_BUILD_PUSH_ACTION = "docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a"; +const HERMES_DEFAULT_TRUST_STEP_NAME = "Verify Hermes default-trust final image"; const HERMES_DOWNLOAD_ARTIFACT_ACTION = "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; const HERMES_UPLOAD_ARTIFACT_ACTION = @@ -470,6 +471,30 @@ function validateGuardedProductionBuild( "Hermes producer must build the production image exactly once with the canonical local-load Buildx action and OS/architecture-scoped GHA cache", ); } + const defaultTrust = requireStep(errors, contract.jobName, job, HERMES_DEFAULT_TRUST_STEP_NAME); + const defaultTrustRun = normalizedShell(defaultTrust.run); + const requiredDefaultTrustFragments = [ + "set -euo pipefail", + "docker run --rm --network none --read-only --cap-drop ALL --security-opt no-new-privileges --pids-limit 64 --memory 256m --entrypoint /bin/sh nemoclaw-hermes-production -eu -c", + 'test "$NODE_EXTRA_CA_CERTS" = /usr/local/share/nemoclaw/corporate-ca.pem', + "test ! -e /usr/local/share/nemoclaw/corporate-ca.pem", + "test ! -L /usr/local/share/nemoclaw/corporate-ca.pem", + "test -x /usr/local/bin/hermes", + 'node -e "const tls = require(\\"node:tls\\"); if (tls.rootCertificates.length === 0) process.exit(1); tls.createSecureContext()"', + '/opt/hermes/.venv/bin/python -I -c "import ssl; assert ssl.create_default_context().get_ca_certs()"', + ]; + if ( + steps(job).filter((step) => step.name === HERMES_DEFAULT_TRUST_STEP_NAME).length !== 1 || + defaultTrust.shell !== "bash" || + requiredDefaultTrustFragments.some((fragment) => !defaultTrustRun.includes(fragment)) || + stepIndex(job, action.name ?? "") >= stepIndex(job, HERMES_DEFAULT_TRUST_STEP_NAME) || + stepIndex(job, HERMES_DEFAULT_TRUST_STEP_NAME) >= + stepIndex(job, "Scan completed Hermes image for node-tar") + ) { + errors.push( + "Hermes producer must prove the no-CA final image uses default trust before completed-image scans", + ); + } return; } From fe93fdd091180578cfc16b07cfbfeeb48b538777 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 15:57:57 -0400 Subject: [PATCH 10/19] test(hermes): bound corporate CA image fixture Signed-off-by: Julie Yaunches --- .github/workflows/sandbox-images-and-e2e.yaml | 16 +- .../checks/select-ci-endpoint-ca-roots.mts | 294 ++++++++++++++++++ .../sandbox-images-workflow-boundary.test.ts | 77 ++++- test/select-ci-endpoint-ca-roots.test.ts | 216 +++++++++++++ .../e2e/sandbox-images-workflow-boundary.mts | 63 +++- 5 files changed, 655 insertions(+), 11 deletions(-) create mode 100644 scripts/checks/select-ci-endpoint-ca-roots.mts create mode 100644 test/select-ci-endpoint-ca-roots.test.ts diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 1a1d1793fd1..875e6cfd3ec 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -438,9 +438,12 @@ jobs: run: | set -euo pipefail # curl and Python replace their default roots with this build argument. - corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt - test -s "$corporate_ca_bundle" - corporate_ca_b64="$(base64 -w 0 "$corporate_ca_bundle")" + compact_ca_bundle="$(mktemp)" + trap 'rm -f "$compact_ca_bundle"' EXIT + node --experimental-strip-types scripts/checks/select-ci-endpoint-ca-roots.mts \ + --output "$compact_ca_bundle" + corporate_ca_b64="$(base64 -w 0 "$compact_ca_bundle")" + corporate_ca_sha256="$(sha256sum "$compact_ca_bundle" | cut -d ' ' -f 1)" messaging_plan_b64="$(node --experimental-strip-types scripts/check-messaging-plan-image-boundary.mts plan hermes)" build_args=( -f agents/hermes/Dockerfile @@ -450,6 +453,13 @@ jobs: ) scripts/check-production-build-args.sh "${build_args[@]}" docker build "${build_args[@]}" -t nemoclaw-hermes-plan-boundary . + installed_ca_sha256="$( + docker run --rm --network none --entrypoint sha256sum \ + nemoclaw-hermes-plan-boundary \ + /usr/local/share/nemoclaw/corporate-ca.pem | + cut -d ' ' -f 1 + )" + test "$installed_ca_sha256" = "$corporate_ca_sha256" docker run --rm --network none --entrypoint openssl \ nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl \ -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null diff --git a/scripts/checks/select-ci-endpoint-ca-roots.mts b/scripts/checks/select-ci-endpoint-ca-roots.mts new file mode 100644 index 00000000000..f40c80cbeab --- /dev/null +++ b/scripts/checks/select-ci-endpoint-ca-roots.mts @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { X509Certificate } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export const CI_CA_SYSTEM_BUNDLE = "/etc/ssl/certs/ca-certificates.crt"; +export const CI_CA_ENDPOINTS = Object.freeze([ + "registry.npmjs.org", + "pypi.org", + "files.pythonhosted.org", +] as const); +export const MAX_CI_CA_CERTIFICATES = 24; +export const MAX_CI_CA_ENCODED_BYTES = 65_536; + +const PEM_RE = /-----BEGIN CERTIFICATE-----\r?\n[A-Za-z0-9+/=\r\n]+?-----END CERTIFICATE-----/gu; +const OPENSSL_TIMEOUT_MS = 30_000; + +type CertificateRecord = { readonly cert: X509Certificate; readonly pem: string }; +type OpenSslResult = { + readonly error?: Error; + readonly status: number | null; + readonly stderr: string; + readonly stdout: string; +}; +export type OpenSslRunner = (args: readonly string[]) => OpenSslResult; + +function runOpenSsl(args: readonly string[]): OpenSslResult { + const result = spawnSync("openssl", [...args], { + encoding: "utf8", + input: "", + killSignal: "SIGKILL", + maxBuffer: 4 * 1024 * 1024, + timeout: OPENSSL_TIMEOUT_MS, + }); + return { + error: result.error, + status: result.status, + stderr: result.stderr ?? "", + stdout: result.stdout ?? "", + }; +} + +function parseCertificates(bundle: string, label: string): CertificateRecord[] { + const blocks = bundle.match(PEM_RE); + if (!blocks?.length) throw new Error(`${label} contains no PEM certificate`); + return blocks.map((pem, index) => { + try { + return { cert: new X509Certificate(pem), pem: pem.trim() }; + } catch { + throw new Error(`${label} certificate ${index + 1} is not valid X.509`); + } + }); +} + +function isSelfSigned(cert: X509Certificate): boolean { + if (cert.subject !== cert.issuer) return false; + try { + return cert.verify(cert.publicKey); + } catch { + return false; + } +} + +function isCurrentSelfSignedRoot(cert: X509Certificate, nowMs = Date.now()): boolean { + const validFromMs = Date.parse(cert.validFrom); + const validToMs = Date.parse(cert.validTo); + if ( + !cert.ca || + !isSelfSigned(cert) || + Number.isNaN(validFromMs) || + Number.isNaN(validToMs) || + nowMs < validFromMs || + nowMs > validToMs + ) { + return false; + } + return true; +} + +function fingerprint(cert: X509Certificate): string { + return cert.fingerprint256.replaceAll(":", "").toLowerCase(); +} + +export function normalizeCompactRootBundle( + roots: readonly string[], + limits: { readonly certificates: number; readonly encodedBytes: number } = { + certificates: MAX_CI_CA_CERTIFICATES, + encodedBytes: MAX_CI_CA_ENCODED_BYTES, + }, +): string { + const unique = new Map(); + for (const [index, pem] of roots.entries()) { + const records = parseCertificates(pem, `selected root ${index + 1}`); + if (records.length !== 1 || !isCurrentSelfSignedRoot(records[0].cert)) { + throw new Error(`selected root ${index + 1} must be a current self-signed CA:TRUE root`); + } + unique.set(fingerprint(records[0].cert), records[0]); + } + if (unique.size === 0) throw new Error("selected root bundle is empty"); + if (unique.size > limits.certificates) { + throw new Error(`selected root bundle exceeds ${limits.certificates} certificates`); + } + const bundle = `${[...unique.values()].map(({ pem }) => pem).join("\n")}\n`; + if (Buffer.from(bundle).toString("base64").length > limits.encodedBytes) { + throw new Error(`selected root bundle exceeds ${limits.encodedBytes} encoded bytes`); + } + return bundle; +} + +function opensslOutput( + runner: OpenSslRunner, + args: readonly string[], + label: string, + requireVerifyOk = false, +): string { + const result = runner(args); + if (result.error || result.status !== 0) { + throw new Error(`${label} failed without emitting certificate data`); + } + const output = `${result.stdout}\n${result.stderr}`; + if (requireVerifyOk && !/Verify return code:\s*0\s*\(ok\)/iu.test(output)) { + throw new Error(`${label} did not report successful certificate verification`); + } + return output; +} + +function connectionArgs(endpoint: string, caFile: string, showCerts: boolean): string[] { + return [ + "s_client", + "-connect", + `${endpoint}:443`, + "-servername", + endpoint, + "-verify_hostname", + endpoint, + "-verify_return_error", + "-CAfile", + caFile, + "-no-CApath", + "-no-CAstore", + ...(showCerts ? ["-showcerts"] : []), + ]; +} + +function systemRoots(systemBundle: string): CertificateRecord[] { + const roots = new Map(); + for (const record of parseCertificates(systemBundle, "system CA bundle")) { + if (isCurrentSelfSignedRoot(record.cert)) roots.set(fingerprint(record.cert), record); + } + if (roots.size === 0) throw new Error("system CA bundle contains no current CA:TRUE root"); + return [...roots.values()]; +} + +function verifiesOffline( + runner: OpenSslRunner, + endpoint: string, + chain: readonly CertificateRecord[], + root: CertificateRecord, + tempDir: string, +): boolean { + const stem = path.join(tempDir, endpoint); + const leaf = `${stem}-leaf.pem`; + const intermediates = `${stem}-intermediates.pem`; + const rootFile = `${stem}-root.pem`; + fs.writeFileSync(leaf, `${chain[0].pem}\n`, { mode: 0o600 }); + fs.writeFileSync(rootFile, `${root.pem}\n`, { mode: 0o600 }); + const untrusted = chain.slice(1).filter(({ cert }) => !isSelfSigned(cert)); + if (untrusted.length) { + fs.writeFileSync(intermediates, `${untrusted.map(({ pem }) => pem).join("\n")}\n`, { + mode: 0o600, + }); + } + const result = runner([ + "verify", + "-purpose", + "sslserver", + "-verify_hostname", + endpoint, + "-CAfile", + rootFile, + "-no-CApath", + "-no-CAstore", + ...(untrusted.length ? ["-untrusted", intermediates] : []), + leaf, + ]); + return !result.error && result.status === 0; +} + +function selectRoot( + runner: OpenSslRunner, + endpoint: string, + chain: readonly CertificateRecord[], + roots: readonly CertificateRecord[], + tempDir: string, +): CertificateRecord { + const issuer = chain.filter(({ cert }) => !isSelfSigned(cert)).at(-1)?.cert.issuer; + const candidates = roots + .filter(({ cert }) => cert.subject === issuer) + .sort((left, right) => fingerprint(left.cert).localeCompare(fingerprint(right.cert))); + const selected = candidates.find((root) => + verifiesOffline(runner, endpoint, chain, root, tempDir), + ); + if (!selected) throw new Error(`no system CA root verifies the chain for ${endpoint}`); + return selected; +} + +function writeOutput(outputPath: string, bundle: string): void { + const stat = fs.lstatSync(outputPath); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error("output must be an existing regular file that is not a symlink"); + } + const fd = fs.openSync( + outputPath, + fs.constants.O_WRONLY | fs.constants.O_TRUNC | fs.constants.O_NOFOLLOW, + ); + try { + fs.writeFileSync(fd, bundle); + fs.fchmodSync(fd, 0o600); + } finally { + fs.closeSync(fd); + } +} + +export function selectCiEndpointCaRoots( + outputPath: string, + runner: OpenSslRunner = runOpenSsl, +): { readonly certificates: number; readonly encodedBytes: number } { + if (path.resolve(outputPath) === path.resolve(CI_CA_SYSTEM_BUNDLE)) { + throw new Error("output must not replace the system CA bundle"); + } + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ci-ca-roots-")); + try { + opensslOutput(runner, ["version"], "OpenSSL availability check"); + const roots = systemRoots(fs.readFileSync(CI_CA_SYSTEM_BUNDLE, "utf8")); + const selected = CI_CA_ENDPOINTS.map((endpoint) => { + const chainOutput = opensslOutput( + runner, + connectionArgs(endpoint, CI_CA_SYSTEM_BUNDLE, true), + `system CA verification for ${endpoint}`, + true, + ); + return selectRoot( + runner, + endpoint, + parseCertificates(chainOutput, `server chain for ${endpoint}`), + roots, + tempDir, + ); + }); + const bundle = normalizeCompactRootBundle(selected.map(({ pem }) => pem)); + const compactPath = path.join(tempDir, "compact.pem"); + fs.writeFileSync(compactPath, bundle, { mode: 0o600 }); + for (const endpoint of CI_CA_ENDPOINTS) { + opensslOutput( + runner, + connectionArgs(endpoint, compactPath, false), + `compact CA verification for ${endpoint}`, + true, + ); + } + writeOutput(outputPath, bundle); + return { + certificates: parseCertificates(bundle, "compact CA bundle").length, + encodedBytes: Buffer.from(bundle).toString("base64").length, + }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +function main(argv: readonly string[]): void { + if (argv.length !== 2 || argv[0] !== "--output" || !argv[1]) { + throw new Error("usage: select-ci-endpoint-ca-roots.mts --output "); + } + const result = selectCiEndpointCaRoots(argv[1]); + process.stdout.write( + `Selected CA roots: ${result.certificates} (${result.encodedBytes} encoded bytes).\n`, + ); +} + +const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ""; +if (invokedPath === import.meta.url) { + try { + main(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`ERROR: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/test/e2e/support/sandbox-images-workflow-boundary.test.ts b/test/e2e/support/sandbox-images-workflow-boundary.test.ts index 7da4e1e6586..1fbf0a9dc2a 100644 --- a/test/e2e/support/sandbox-images-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-images-workflow-boundary.test.ts @@ -451,8 +451,8 @@ describe("sandbox image workflow boundary", () => { )!; hermes.run = hermes .run!.replace( - "corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt", - "corporate_ca_bundle=/missing/ca-certificates.crt", + 'scripts/check-production-build-args.sh "${build_args[@]}"', + 'echo "guard bypassed"', ) .replace( '--build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"', @@ -471,7 +471,7 @@ describe("sandbox image workflow boundary", () => { "messaging plan image boundary must set up Node exactly once", "messaging plan image boundary must use Node 22.19.0", 'openclaw messaging plan image boundary must include scripts/check-production-build-args.sh "${build_args[@]}"', - 'hermes messaging plan image boundary must include corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt test -s "$corporate_ca_bundle" corporate_ca_b64="$(base64 -w 0 "$corporate_ca_bundle")"', + 'hermes messaging plan image boundary must include scripts/check-production-build-args.sh "${build_args[@]}"', 'hermes messaging plan image boundary must include --build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"', "hermes messaging plan image boundary must include docker run --rm --network none --entrypoint openssl nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null", "hermes messaging plan image boundary must include node --experimental-strip-types scripts/check-messaging-plan-image-boundary.mts verify nemoclaw-hermes-plan-boundary hermes", @@ -480,6 +480,77 @@ describe("sandbox image workflow boundary", () => { ); }); + it("requires the exact compact CA root helper invocation", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const hermes = imageWorkflow.jobs["messaging-plan-image-boundary"].steps!.find( + (step) => step.name === "Build and verify Hermes messaging plan boundary", + )!; + hermes.run = hermes.run!.replace( + "select-ci-endpoint-ca-roots.mts", + "select-ci-endpoint-ca-roots.mts --endpoint registry.example.invalid", + ); + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + 'hermes messaging plan image boundary must include exactly node --experimental-strip-types scripts/checks/select-ci-endpoint-ca-roots.mts --output "$compact_ca_bundle"', + ); + }); + + it("rejects direct base64 encoding of the broad system CA bundle", () => { + for (const forbidden of [ + 'forbidden_ca_b64="$(base64 -w 0 "$system_ca_bundle")"', + [ + "corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt", + 'forbidden_ca_b64="$(base64 -w 0 "$corporate_ca_bundle")"', + ].join("\n"), + ]) { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const hermes = imageWorkflow.jobs["messaging-plan-image-boundary"].steps!.find( + (step) => step.name === "Build and verify Hermes messaging plan boundary", + )!; + hermes.run = `${hermes.run}\n${forbidden}`; + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + "hermes messaging plan image boundary must not encode the system CA bundle directly", + ); + } + }); + + it("requires offline equality and parse proofs for the installed Hermes CA bundle", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const hermes = imageWorkflow.jobs["messaging-plan-image-boundary"].steps!.find( + (step) => step.name === "Build and verify Hermes messaging plan boundary", + )!; + hermes.run = hermes + .run!.replace( + 'test "$installed_ca_sha256" = "$corporate_ca_sha256"', + 'test -n "$installed_ca_sha256"', + ) + .replace("crl2pkcs7 -nocrl", "version"); + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( + expect.arrayContaining([ + 'hermes messaging plan image boundary must include test "$installed_ca_sha256" = "$corporate_ca_sha256"', + "hermes messaging plan image boundary must include docker run --rm --network none --entrypoint openssl nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null", + ]), + ); + }); + + it("requires the Hermes build guard before the build and offline CA proofs", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const hermes = imageWorkflow.jobs["messaging-plan-image-boundary"].steps!.find( + (step) => step.name === "Build and verify Hermes messaging plan boundary", + )!; + const guard = 'scripts/check-production-build-args.sh "${build_args[@]}"'; + hermes.run = `${hermes.run!.replace(guard, "")}\n${guard}`; + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toEqual( + expect.arrayContaining([ + "hermes messaging plan image boundary steps are out of order", + "hermes messaging plan image boundary CA fixture steps are out of order", + ]), + ); + }); + it("requires bounded swap before every hosted Hermes image export", () => { const { imageWorkflow, mainWorkflow } = readWorkflows(); for (const jobName of ["build-hermes-sandbox-image", "messaging-plan-image-boundary"]) { diff --git a/test/select-ci-endpoint-ca-roots.test.ts b/test/select-ci-endpoint-ca-roots.test.ts new file mode 100644 index 00000000000..b3a0bb41a33 --- /dev/null +++ b/test/select-ci-endpoint-ca-roots.test.ts @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + CI_CA_ENDPOINTS, + CI_CA_SYSTEM_BUNDLE, + MAX_CI_CA_CERTIFICATES, + MAX_CI_CA_ENCODED_BYTES, + normalizeCompactRootBundle, + type OpenSslRunner, + selectCiEndpointCaRoots, +} from "../scripts/checks/select-ci-endpoint-ca-roots.mts"; +import { LEAF_PEM, PEM, tmpDir } from "../src/lib/onboard/__test-helpers__/corporate-ca-fixtures"; + +const hasOpenSsl = spawnSync("openssl", ["version"], { encoding: "utf8" }).status === 0; + +function openssl(args: readonly string[], cwd: string): void { + const result = spawnSync("openssl", [...args], { + cwd, + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 10_000, + }); + if (result.status !== 0) throw new Error(`OpenSSL fixture command failed: ${args[0]}`); +} + +function createEndpointCertificate(directory: string): { leaf: string; root: string } { + fs.writeFileSync( + path.join(directory, "leaf.ext"), + [ + "basicConstraints=critical,CA:FALSE", + "keyUsage=critical,digitalSignature,keyEncipherment", + "extendedKeyUsage=serverAuth", + `subjectAltName=${CI_CA_ENDPOINTS.map((endpoint) => `DNS:${endpoint}`).join(",")}`, + "", + ].join("\n"), + ); + openssl( + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=NemoClaw CI Root", + "-keyout", + "root.key", + "-out", + "root.pem", + "-days", + "2", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ], + directory, + ); + openssl( + [ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + `/CN=${CI_CA_ENDPOINTS[0]}`, + "-keyout", + "leaf.key", + "-out", + "leaf.csr", + ], + directory, + ); + openssl( + [ + "x509", + "-req", + "-in", + "leaf.csr", + "-CA", + "root.pem", + "-CAkey", + "root.key", + "-CAcreateserial", + "-out", + "leaf.pem", + "-days", + "2", + "-extfile", + "leaf.ext", + ], + directory, + ); + return { + leaf: fs.readFileSync(path.join(directory, "leaf.pem"), "utf8"), + root: fs.readFileSync(path.join(directory, "root.pem"), "utf8"), + }; +} + +describe("CI endpoint CA root selection", () => { + it("keeps the endpoint set and build-argument limits fixed", () => { + expect(CI_CA_SYSTEM_BUNDLE).toBe("/etc/ssl/certs/ca-certificates.crt"); + expect(CI_CA_ENDPOINTS).toEqual(["registry.npmjs.org", "pypi.org", "files.pythonhosted.org"]); + expect(MAX_CI_CA_CERTIFICATES).toBe(24); + expect(MAX_CI_CA_ENCODED_BYTES).toBe(65_536); + }); + + it("deduplicates CA roots and rejects leaf certificates or oversized output", () => { + expect(normalizeCompactRootBundle([PEM, PEM])).toBe(PEM); + expect(() => normalizeCompactRootBundle([LEAF_PEM])).toThrow(/CA:TRUE root/u); + expect(() => + normalizeCompactRootBundle([PEM], { certificates: 0, encodedBytes: 65_536 }), + ).toThrow(/exceeds 0 certificates/u); + expect(() => normalizeCompactRootBundle([PEM], { certificates: 24, encodedBytes: 1 })).toThrow( + /exceeds 1 encoded bytes/u, + ); + }); + + it.skipIf(!hasOpenSsl)( + "selects one system root after offline hostname verification for every endpoint", + () => { + const directory = tmpDir(); + const output = path.join(directory, "compact.pem"); + fs.writeFileSync(output, "", { mode: 0o600 }); + const fixture = createEndpointCertificate(directory); + const realReadFile = fs.readFileSync.bind(fs); + vi.spyOn(fs, "readFileSync").mockImplementation(((file, ...args) => { + if (file === CI_CA_SYSTEM_BUNDLE) return fixture.root; + return realReadFile(file, ...args); + }) as typeof fs.readFileSync); + + const connections: string[][] = []; + const runner: OpenSslRunner = (args) => { + if (args[0] === "s_client") { + connections.push([...args]); + return { + status: 0, + stderr: "", + stdout: `${fixture.leaf}\nVerify return code: 0 (ok)\n`, + }; + } + const result = spawnSync("openssl", [...args], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 10_000, + }); + return { + error: result.error, + status: result.status, + stderr: result.stderr ?? "", + stdout: result.stdout ?? "", + }; + }; + + expect(selectCiEndpointCaRoots(output, runner)).toEqual({ + certificates: 1, + encodedBytes: Buffer.from(fixture.root).toString("base64").length, + }); + expect(fs.readFileSync(output, "utf8")).toBe(fixture.root); + expect(connections).toHaveLength(CI_CA_ENDPOINTS.length * 2); + for (const endpoint of CI_CA_ENDPOINTS) { + const endpointConnections = connections.filter((args) => args.includes(`${endpoint}:443`)); + expect(endpointConnections).toEqual([ + [ + "s_client", + "-connect", + `${endpoint}:443`, + "-servername", + endpoint, + "-verify_hostname", + endpoint, + "-verify_return_error", + "-CAfile", + CI_CA_SYSTEM_BUNDLE, + "-no-CApath", + "-no-CAstore", + "-showcerts", + ], + [ + "s_client", + "-connect", + `${endpoint}:443`, + "-servername", + endpoint, + "-verify_hostname", + endpoint, + "-verify_return_error", + "-CAfile", + expect.stringMatching(/\/compact\.pem$/u), + "-no-CApath", + "-no-CAstore", + ], + ]); + } + + fs.writeFileSync(output, "unchanged", { mode: 0o600 }); + const rejectCompactVerification: OpenSslRunner = (args) => { + if (args[0] === "s_client" && !args.includes("-showcerts")) { + return { status: 1, stderr: "verification failed", stdout: "" }; + } + return runner(args); + }; + expect(() => selectCiEndpointCaRoots(output, rejectCompactVerification)).toThrow( + /compact CA verification for registry\.npmjs\.org failed/u, + ); + expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); + }, + ); +}); diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index 5214e080699..5673b2a30f5 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -666,6 +666,62 @@ function validateMessagingPlanBoundaryBuild( } } +function validateHermesMessagingPlanCaFixture( + errors: string[], + job: SandboxImagesWorkflowJob, +): void { + const step = findStep(job, "Build and verify Hermes messaging plan boundary"); + const run = normalizedShell(step?.run); + const helperInvocation = + 'node --experimental-strip-types scripts/checks/select-ci-endpoint-ca-roots.mts --output "$compact_ca_bundle"'; + const compactEncoding = 'corporate_ca_b64="$(base64 -w 0 "$compact_ca_bundle")"'; + const sourceHash = 'corporate_ca_sha256="$(sha256sum "$compact_ca_bundle" | cut -d \' \' -f 1)"'; + const installedHash = + "installed_ca_sha256=\"$( docker run --rm --network none --entrypoint sha256sum nemoclaw-hermes-plan-boundary /usr/local/share/nemoclaw/corporate-ca.pem | cut -d ' ' -f 1 )\""; + const matchingHash = 'test "$installed_ca_sha256" = "$corporate_ca_sha256"'; + const parseProof = + "docker run --rm --network none --entrypoint openssl nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null"; + const orderedFragments = [ + 'compact_ca_bundle="$(mktemp)"', + "trap 'rm -f \"$compact_ca_bundle\"' EXIT", + helperInvocation, + compactEncoding, + sourceHash, + "check-messaging-plan-image-boundary.mts plan", + "check-production-build-args.sh", + 'docker build "${build_args[@]}" -t nemoclaw-hermes-plan-boundary', + installedHash, + matchingHash, + parseProof, + "check-messaging-plan-image-boundary.mts verify", + ]; + for (const fragment of orderedFragments) { + if (!run.includes(fragment)) { + errors.push(`hermes messaging plan image boundary must include ${fragment}`); + } + } + if ( + run.split("select-ci-endpoint-ca-roots.mts").length - 1 !== 1 || + !run.includes(`${helperInvocation} ${compactEncoding}`) + ) { + errors.push(`hermes messaging plan image boundary must include exactly ${helperInvocation}`); + } + if ( + run.includes("/etc/ssl/certs/ca-certificates.crt") || + /base64 -w 0 "?\$\{?system_ca_bundle\}?"?/u.test(run) + ) { + errors.push( + "hermes messaging plan image boundary must not encode the system CA bundle directly", + ); + } + const positions = orderedFragments.map((fragment) => run.indexOf(fragment)); + if ( + positions.some((position, index) => position < 0 || position <= (positions[index - 1] ?? -1)) + ) { + errors.push("hermes messaging plan image boundary CA fixture steps are out of order"); + } +} + function validateMessagingPlanImageBoundary( errors: string[], workflow: SandboxImagesWorkflow, @@ -704,14 +760,11 @@ function validateMessagingPlanImageBoundary( agent: "hermes", baseArgName: "BASE_IMAGE", baseEnvName: "HERMES_BASE_IMAGE", - extraRequiredFragments: [ - 'corporate_ca_bundle=/etc/ssl/certs/ca-certificates.crt test -s "$corporate_ca_bundle" corporate_ca_b64="$(base64 -w 0 "$corporate_ca_bundle")"', - '--build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"', - "docker run --rm --network none --entrypoint openssl nemoclaw-hermes-plan-boundary crl2pkcs7 -nocrl -certfile /usr/local/share/nemoclaw/corporate-ca.pem -out /dev/null", - ], + extraRequiredFragments: ['--build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"'], stepName: "Build and verify Hermes messaging plan boundary", target: "nemoclaw-hermes-plan-boundary", }); + validateHermesMessagingPlanCaFixture(errors, job); const builds = dockerBuildLines(job); if ( From 591aaac7586e4117e020e3a8257a314c4ba2b761 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 16:08:00 -0400 Subject: [PATCH 11/19] test(hermes): keep CA fixture test linear Signed-off-by: Julie Yaunches --- test/select-ci-endpoint-ca-roots.test.ts | 40 ++++++++++++------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/select-ci-endpoint-ca-roots.test.ts b/test/select-ci-endpoint-ca-roots.test.ts index b3a0bb41a33..dd2b309ecf9 100644 --- a/test/select-ci-endpoint-ca-roots.test.ts +++ b/test/select-ci-endpoint-ca-roots.test.ts @@ -27,7 +27,7 @@ function openssl(args: readonly string[], cwd: string): void { killSignal: "SIGKILL", timeout: 10_000, }); - if (result.status !== 0) throw new Error(`OpenSSL fixture command failed: ${args[0]}`); + expect(result.status, `OpenSSL fixture command failed: ${args[0]}`).toBe(0); } function createEndpointCertificate(directory: string): { leaf: string; root: string } { @@ -131,21 +131,21 @@ describe("CI endpoint CA root selection", () => { fs.writeFileSync(output, "", { mode: 0o600 }); const fixture = createEndpointCertificate(directory); const realReadFile = fs.readFileSync.bind(fs); - vi.spyOn(fs, "readFileSync").mockImplementation(((file, ...args) => { - if (file === CI_CA_SYSTEM_BUNDLE) return fixture.root; - return realReadFile(file, ...args); - }) as typeof fs.readFileSync); + vi.spyOn(fs, "readFileSync").mockImplementation(((file, ...args) => + file === CI_CA_SYSTEM_BUNDLE + ? fixture.root + : realReadFile(file, ...args)) as typeof fs.readFileSync); const connections: string[][] = []; - const runner: OpenSslRunner = (args) => { - if (args[0] === "s_client") { - connections.push([...args]); - return { - status: 0, - stderr: "", - stdout: `${fixture.leaf}\nVerify return code: 0 (ok)\n`, - }; - } + const runConnection: OpenSslRunner = (args) => { + connections.push([...args]); + return { + status: 0, + stderr: "", + stdout: `${fixture.leaf}\nVerify return code: 0 (ok)\n`, + }; + }; + const runActualOpenSsl: OpenSslRunner = (args) => { const result = spawnSync("openssl", [...args], { encoding: "utf8", killSignal: "SIGKILL", @@ -158,6 +158,8 @@ describe("CI endpoint CA root selection", () => { stdout: result.stdout ?? "", }; }; + const runner: OpenSslRunner = (args) => + args[0] === "s_client" ? runConnection(args) : runActualOpenSsl(args); expect(selectCiEndpointCaRoots(output, runner)).toEqual({ certificates: 1, @@ -201,12 +203,10 @@ describe("CI endpoint CA root selection", () => { } fs.writeFileSync(output, "unchanged", { mode: 0o600 }); - const rejectCompactVerification: OpenSslRunner = (args) => { - if (args[0] === "s_client" && !args.includes("-showcerts")) { - return { status: 1, stderr: "verification failed", stdout: "" }; - } - return runner(args); - }; + const rejectCompactVerification: OpenSslRunner = (args) => + args[0] === "s_client" && !args.includes("-showcerts") + ? { status: 1, stderr: "verification failed", stdout: "" } + : runner(args); expect(() => selectCiEndpointCaRoots(output, rejectCompactVerification)).toThrow( /compact CA verification for registry\.npmjs\.org failed/u, ); From 6b96ddbc2fb63834238fd924ff014375ec690ee9 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 16:18:49 -0400 Subject: [PATCH 12/19] fix(ci): select cross-signed CA trust path Signed-off-by: Julie Yaunches --- .../checks/select-ci-endpoint-ca-roots.mts | 15 ++-- test/select-ci-endpoint-ca-roots.test.ts | 82 ++++++++++++++++++- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/scripts/checks/select-ci-endpoint-ca-roots.mts b/scripts/checks/select-ci-endpoint-ca-roots.mts index f40c80cbeab..6ca88b8d0d9 100644 --- a/scripts/checks/select-ci-endpoint-ca-roots.mts +++ b/scripts/checks/select-ci-endpoint-ca-roots.mts @@ -57,15 +57,18 @@ function parseCertificates(bundle: string, label: string): CertificateRecord[] { }); } -function isSelfSigned(cert: X509Certificate): boolean { - if (cert.subject !== cert.issuer) return false; +function isSignedBy(cert: X509Certificate, issuer: X509Certificate): boolean { try { - return cert.verify(cert.publicKey); + return cert.verify(issuer.publicKey); } catch { return false; } } +function isSelfSigned(cert: X509Certificate): boolean { + return cert.subject === cert.issuer && isSignedBy(cert, cert); +} + function isCurrentSelfSignedRoot(cert: X509Certificate, nowMs = Date.now()): boolean { const validFromMs = Date.parse(cert.validFrom); const validToMs = Date.parse(cert.validTo); @@ -198,9 +201,11 @@ function selectRoot( roots: readonly CertificateRecord[], tempDir: string, ): CertificateRecord { - const issuer = chain.filter(({ cert }) => !isSelfSigned(cert)).at(-1)?.cert.issuer; + const untrusted = chain.filter(({ cert }) => !isSelfSigned(cert)); const candidates = roots - .filter(({ cert }) => cert.subject === issuer) + .filter(({ cert: root }) => + untrusted.some(({ cert }) => cert.issuer === root.subject && isSignedBy(cert, root)), + ) .sort((left, right) => fingerprint(left.cert).localeCompare(fingerprint(right.cert))); const selected = candidates.find((root) => verifiesOffline(runner, endpoint, chain, root, tempDir), diff --git a/test/select-ci-endpoint-ca-roots.test.ts b/test/select-ci-endpoint-ca-roots.test.ts index dd2b309ecf9..fcf6a3c67ed 100644 --- a/test/select-ci-endpoint-ca-roots.test.ts +++ b/test/select-ci-endpoint-ca-roots.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { X509Certificate } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; @@ -30,7 +31,21 @@ function openssl(args: readonly string[], cwd: string): void { expect(result.status, `OpenSSL fixture command failed: ${args[0]}`).toBe(0); } -function createEndpointCertificate(directory: string): { leaf: string; root: string } { +function createEndpointCertificate(directory: string): { + chain: string; + crossSignedRoot: string; + root: string; +} { + fs.writeFileSync( + path.join(directory, "root.ext"), + [ + "basicConstraints=critical,CA:TRUE", + "keyUsage=critical,keyCertSign,cRLSign", + "subjectKeyIdentifier=hash", + "authorityKeyIdentifier=keyid,issuer", + "", + ].join("\n"), + ); fs.writeFileSync( path.join(directory, "leaf.ext"), [ @@ -63,6 +78,52 @@ function createEndpointCertificate(directory: string): { leaf: string; root: str ], directory, ); + openssl( + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=NemoClaw Alternate Root", + "-keyout", + "alternate-root.key", + "-out", + "alternate-root.pem", + "-days", + "2", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ], + directory, + ); + openssl( + ["req", "-new", "-key", "root.key", "-subj", "/CN=NemoClaw CI Root", "-out", "root.csr"], + directory, + ); + openssl( + [ + "x509", + "-req", + "-in", + "root.csr", + "-CA", + "alternate-root.pem", + "-CAkey", + "alternate-root.key", + "-CAcreateserial", + "-out", + "root-cross-signed.pem", + "-days", + "2", + "-extfile", + "root.ext", + ], + directory, + ); openssl( [ "req", @@ -98,8 +159,13 @@ function createEndpointCertificate(directory: string): { leaf: string; root: str ], directory, ); + const leaf = fs.readFileSync(path.join(directory, "leaf.pem"), "utf8").trim(); + const crossSignedRoot = fs + .readFileSync(path.join(directory, "root-cross-signed.pem"), "utf8") + .trim(); return { - leaf: fs.readFileSync(path.join(directory, "leaf.pem"), "utf8"), + chain: `${leaf}\n${crossSignedRoot}\n`, + crossSignedRoot, root: fs.readFileSync(path.join(directory, "root.pem"), "utf8"), }; } @@ -124,12 +190,20 @@ describe("CI endpoint CA root selection", () => { }); it.skipIf(!hasOpenSsl)( - "selects one system root after offline hostname verification for every endpoint", + "selects a self-signed system root when the server sends its cross-signed form", () => { const directory = tmpDir(); const output = path.join(directory, "compact.pem"); fs.writeFileSync(output, "", { mode: 0o600 }); const fixture = createEndpointCertificate(directory); + const systemRoot = new X509Certificate(fixture.root); + const crossSignedRoot = new X509Certificate(fixture.crossSignedRoot); + expect(crossSignedRoot.subject).toBe(systemRoot.subject); + expect(crossSignedRoot.issuer).not.toBe(crossSignedRoot.subject); + expect(crossSignedRoot.publicKey.export({ format: "der", type: "spki" })).toEqual( + systemRoot.publicKey.export({ format: "der", type: "spki" }), + ); + expect(crossSignedRoot.verify(crossSignedRoot.publicKey)).toBe(false); const realReadFile = fs.readFileSync.bind(fs); vi.spyOn(fs, "readFileSync").mockImplementation(((file, ...args) => file === CI_CA_SYSTEM_BUNDLE @@ -142,7 +216,7 @@ describe("CI endpoint CA root selection", () => { return { status: 0, stderr: "", - stdout: `${fixture.leaf}\nVerify return code: 0 (ok)\n`, + stdout: `${fixture.chain}Verify return code: 0 (ok)\n`, }; }; const runActualOpenSsl: OpenSslRunner = (args) => { From bad46836704e4706951fd1ec9a52251872d52b18 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 4 Aug 2026 16:37:14 -0400 Subject: [PATCH 13/19] fix(ci): eliminate CA fixture output race Signed-off-by: Julie Yaunches --- scripts/checks/select-ci-endpoint-ca-roots.mts | 17 ++++++++++------- test/select-ci-endpoint-ca-roots.test.ts | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/scripts/checks/select-ci-endpoint-ca-roots.mts b/scripts/checks/select-ci-endpoint-ca-roots.mts index 6ca88b8d0d9..4501e90b132 100644 --- a/scripts/checks/select-ci-endpoint-ca-roots.mts +++ b/scripts/checks/select-ci-endpoint-ca-roots.mts @@ -215,15 +215,18 @@ function selectRoot( } function writeOutput(outputPath: string, bundle: string): void { - const stat = fs.lstatSync(outputPath); - if (!stat.isFile() || stat.isSymbolicLink()) { - throw new Error("output must be an existing regular file that is not a symlink"); + let fd: number; + try { + fd = fs.openSync(outputPath, fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW); + } catch { + throw new Error("output must be an existing single-link regular file"); } - const fd = fs.openSync( - outputPath, - fs.constants.O_WRONLY | fs.constants.O_TRUNC | fs.constants.O_NOFOLLOW, - ); try { + const stat = fs.fstatSync(fd); + if (!stat.isFile() || stat.nlink !== 1) { + throw new Error("output must be an existing single-link regular file"); + } + fs.ftruncateSync(fd, 0); fs.writeFileSync(fd, bundle); fs.fchmodSync(fd, 0o600); } finally { diff --git a/test/select-ci-endpoint-ca-roots.test.ts b/test/select-ci-endpoint-ca-roots.test.ts index fcf6a3c67ed..d96e1c9895c 100644 --- a/test/select-ci-endpoint-ca-roots.test.ts +++ b/test/select-ci-endpoint-ca-roots.test.ts @@ -285,6 +285,22 @@ describe("CI endpoint CA root selection", () => { /compact CA verification for registry\.npmjs\.org failed/u, ); expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); + + const hardLink = path.join(directory, "compact-hard-link.pem"); + fs.linkSync(output, hardLink); + expect(() => selectCiEndpointCaRoots(output, runner)).toThrow( + /output must be an existing single-link regular file/u, + ); + expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); + expect(fs.readFileSync(hardLink, "utf8")).toBe("unchanged"); + fs.unlinkSync(hardLink); + + const symlink = path.join(directory, "compact-symlink.pem"); + fs.symlinkSync(output, symlink); + expect(() => selectCiEndpointCaRoots(symlink, runner)).toThrow( + /output must be an existing single-link regular file/u, + ); + expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); }, ); }); From 47c2ff145c8172ba696b3cb97a9bdc33b7d383d7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 14:11:28 -0700 Subject: [PATCH 14/19] fix(ci): validate CA fixture file and build order Signed-off-by: Carlos Villela --- .../checks/select-ci-endpoint-ca-roots.mts | 42 ++++-- .../sandbox-images-workflow-boundary.test.ts | 17 +++ test/select-ci-endpoint-ca-roots.test.ts | 132 +++++++++++++++--- .../e2e/sandbox-images-workflow-boundary.mts | 2 + 4 files changed, 169 insertions(+), 24 deletions(-) diff --git a/scripts/checks/select-ci-endpoint-ca-roots.mts b/scripts/checks/select-ci-endpoint-ca-roots.mts index 4501e90b132..f278d2e400d 100644 --- a/scripts/checks/select-ci-endpoint-ca-roots.mts +++ b/scripts/checks/select-ci-endpoint-ca-roots.mts @@ -214,17 +214,43 @@ function selectRoot( return selected; } -function writeOutput(outputPath: string, bundle: string): void { +export function writeCiEndpointCaRootsOutput(outputPath: string, bundle: string): void { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + throw new Error("output requires O_NOFOLLOW support"); + } + + const beforeOpen = fs.lstatSync(outputPath); + if (!beforeOpen.isFile() || beforeOpen.isSymbolicLink() || beforeOpen.nlink !== 1) { + throw new Error("output must be an existing regular file with exactly one link"); + } + let fd: number; try { - fd = fs.openSync(outputPath, fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW); - } catch { - throw new Error("output must be an existing single-link regular file"); + fd = fs.openSync( + outputPath, + fs.constants.O_WRONLY | noFollow | (fs.constants.O_NONBLOCK ?? 0), + ); + } catch (error) { + throw new Error("output must be an existing regular file that is not a symlink", { + cause: error, + }); } try { - const stat = fs.fstatSync(fd); - if (!stat.isFile() || stat.nlink !== 1) { - throw new Error("output must be an existing single-link regular file"); + const opened = fs.fstatSync(fd); + const afterOpen = fs.lstatSync(outputPath); + if ( + !opened.isFile() || + opened.nlink !== 1 || + !afterOpen.isFile() || + afterOpen.isSymbolicLink() || + afterOpen.nlink !== 1 || + beforeOpen.dev !== opened.dev || + beforeOpen.ino !== opened.ino || + opened.dev !== afterOpen.dev || + opened.ino !== afterOpen.ino + ) { + throw new Error("output must remain the same regular file with exactly one link"); } fs.ftruncateSync(fd, 0); fs.writeFileSync(fd, bundle); @@ -271,7 +297,7 @@ export function selectCiEndpointCaRoots( true, ); } - writeOutput(outputPath, bundle); + writeCiEndpointCaRootsOutput(outputPath, bundle); return { certificates: parseCertificates(bundle, "compact CA bundle").length, encodedBytes: Buffer.from(bundle).toString("base64").length, diff --git a/test/e2e/support/sandbox-images-workflow-boundary.test.ts b/test/e2e/support/sandbox-images-workflow-boundary.test.ts index 1fbf0a9dc2a..97207ead1c7 100644 --- a/test/e2e/support/sandbox-images-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-images-workflow-boundary.test.ts @@ -551,6 +551,23 @@ describe("sandbox image workflow boundary", () => { ); }); + it("rejects the Hermes CA build argument after the image build", () => { + const { imageWorkflow, mainWorkflow } = readWorkflows(); + const hermes = imageWorkflow.jobs["messaging-plan-image-boundary"].steps!.find( + (step) => step.name === "Build and verify Hermes messaging plan boundary", + )!; + const buildArg = '--build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"'; + const buildCommand = 'docker build "${build_args[@]}" -t nemoclaw-hermes-plan-boundary .'; + expect(hermes.run).toContain(buildArg); + hermes.run = hermes + .run!.replace(buildArg, "") + .replace(buildCommand, `${buildCommand}\n${buildArg}`); + + expect(validateSandboxImagesWorkflow(imageWorkflow, mainWorkflow)).toContain( + "hermes messaging plan image boundary CA fixture steps are out of order", + ); + }); + it("requires bounded swap before every hosted Hermes image export", () => { const { imageWorkflow, mainWorkflow } = readWorkflows(); for (const jobName of ["build-hermes-sandbox-image", "messaging-plan-image-boundary"]) { diff --git a/test/select-ci-endpoint-ca-roots.test.ts b/test/select-ci-endpoint-ca-roots.test.ts index d96e1c9895c..da45f3ec69c 100644 --- a/test/select-ci-endpoint-ca-roots.test.ts +++ b/test/select-ci-endpoint-ca-roots.test.ts @@ -16,6 +16,7 @@ import { normalizeCompactRootBundle, type OpenSslRunner, selectCiEndpointCaRoots, + writeCiEndpointCaRootsOutput, } from "../scripts/checks/select-ci-endpoint-ca-roots.mts"; import { LEAF_PEM, PEM, tmpDir } from "../src/lib/onboard/__test-helpers__/corporate-ca-fixtures"; @@ -189,6 +190,121 @@ describe("CI endpoint CA root selection", () => { ); }); + it("validates each CA output identity before truncating it", () => { + const output = path.join(tmpDir(), "compact.pem"); + fs.writeFileSync(output, "unchanged", { mode: 0o644 }); + const calls: string[] = []; + const realLstatSync = fs.lstatSync.bind(fs); + const realOpenSync = fs.openSync.bind(fs); + const realFstatSync = fs.fstatSync.bind(fs); + const realFtruncateSync = fs.ftruncateSync.bind(fs); + vi.spyOn(fs, "lstatSync").mockImplementation((file) => { + calls.push("lstat"); + return realLstatSync(file); + }); + vi.spyOn(fs, "openSync").mockImplementation((file, flags, mode) => { + calls.push("open"); + return realOpenSync(file, flags, mode); + }); + vi.spyOn(fs, "fstatSync").mockImplementation((descriptor) => { + calls.push("fstat"); + expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); + return realFstatSync(descriptor); + }); + vi.spyOn(fs, "ftruncateSync").mockImplementation((descriptor, length) => { + calls.push("truncate"); + expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); + return realFtruncateSync(descriptor, length); + }); + + writeCiEndpointCaRootsOutput(output, "replacement"); + + expect(calls).toEqual(["lstat", "open", "fstat", "lstat", "truncate"]); + expect(fs.readFileSync(output, "utf8")).toBe("replacement"); + expect(fs.statSync(output).mode & 0o777).toBe(0o600); + }); + + it.skipIf(process.platform === "win32")("rejects a FIFO CA output before opening it", () => { + const output = path.join(tmpDir(), "compact.pem"); + const created = spawnSync("mkfifo", [output], { encoding: "utf8", timeout: 5_000 }); + expect(created.status, created.stderr).toBe(0); + const openSync = vi.spyOn(fs, "openSync"); + + expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( + "output must be an existing regular file with exactly one link", + ); + + expect(openSync).not.toHaveBeenCalled(); + expect(fs.lstatSync(output).isFIFO()).toBe(true); + }); + + it.skipIf(process.platform === "win32")("rejects a symlinked CA output before opening it", () => { + const directory = tmpDir(); + const target = path.join(directory, "target.pem"); + const output = path.join(directory, "compact.pem"); + fs.writeFileSync(target, "target", { mode: 0o640 }); + fs.symlinkSync(target, output); + const openSync = vi.spyOn(fs, "openSync"); + + expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( + "output must be an existing regular file with exactly one link", + ); + + expect(openSync).not.toHaveBeenCalled(); + expect(fs.lstatSync(output).isSymbolicLink()).toBe(true); + expect(fs.readFileSync(target, "utf8")).toBe("target"); + expect(fs.statSync(target).mode & 0o777).toBe(0o640); + }); + + it.skipIf(process.platform === "win32")( + "rejects a hard-linked CA output without changing either path", + () => { + const directory = tmpDir(); + const output = path.join(directory, "compact.pem"); + const linked = path.join(directory, "linked.pem"); + fs.writeFileSync(output, "unchanged", { mode: 0o640 }); + fs.linkSync(output, linked); + const openSync = vi.spyOn(fs, "openSync"); + + expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( + "output must be an existing regular file with exactly one link", + ); + + expect(openSync).not.toHaveBeenCalled(); + for (const file of [output, linked]) { + expect(fs.readFileSync(file, "utf8")).toBe("unchanged"); + expect(fs.statSync(file).mode & 0o777).toBe(0o640); + } + }, + ); + + it("rejects a substituted CA output without changing either file", () => { + const directory = tmpDir(); + const output = path.join(directory, "compact.pem"); + const original = path.join(directory, "original.pem"); + const replacement = path.join(directory, "replacement.pem"); + fs.writeFileSync(output, "original", { mode: 0o640 }); + fs.writeFileSync(replacement, "replacement", { mode: 0o604 }); + const realFstatSync = fs.fstatSync.bind(fs); + vi.spyOn(fs, "fstatSync").mockImplementation((descriptor) => { + const stat = realFstatSync(descriptor); + fs.renameSync(output, original); + fs.renameSync(replacement, output); + return stat; + }); + const ftruncateSync = vi.spyOn(fs, "ftruncateSync"); + + expect(() => writeCiEndpointCaRootsOutput(output, "written")).toThrow( + "output must remain the same regular file with exactly one link", + ); + + expect(ftruncateSync).not.toHaveBeenCalled(); + expect(fs.readFileSync(original, "utf8")).toBe("original"); + expect(fs.statSync(original).mode & 0o777).toBe(0o640); + expect(fs.readFileSync(output, "utf8")).toBe("replacement"); + expect(fs.statSync(output).mode & 0o777).toBe(0o604); + }); + it.skipIf(!hasOpenSsl)( "selects a self-signed system root when the server sends its cross-signed form", () => { @@ -285,22 +401,6 @@ describe("CI endpoint CA root selection", () => { /compact CA verification for registry\.npmjs\.org failed/u, ); expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); - - const hardLink = path.join(directory, "compact-hard-link.pem"); - fs.linkSync(output, hardLink); - expect(() => selectCiEndpointCaRoots(output, runner)).toThrow( - /output must be an existing single-link regular file/u, - ); - expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); - expect(fs.readFileSync(hardLink, "utf8")).toBe("unchanged"); - fs.unlinkSync(hardLink); - - const symlink = path.join(directory, "compact-symlink.pem"); - fs.symlinkSync(output, symlink); - expect(() => selectCiEndpointCaRoots(symlink, runner)).toThrow( - /output must be an existing single-link regular file/u, - ); - expect(fs.readFileSync(output, "utf8")).toBe("unchanged"); }, ); }); diff --git a/tools/e2e/sandbox-images-workflow-boundary.mts b/tools/e2e/sandbox-images-workflow-boundary.mts index 5673b2a30f5..437ae89cf7d 100644 --- a/tools/e2e/sandbox-images-workflow-boundary.mts +++ b/tools/e2e/sandbox-images-workflow-boundary.mts @@ -676,6 +676,7 @@ function validateHermesMessagingPlanCaFixture( 'node --experimental-strip-types scripts/checks/select-ci-endpoint-ca-roots.mts --output "$compact_ca_bundle"'; const compactEncoding = 'corporate_ca_b64="$(base64 -w 0 "$compact_ca_bundle")"'; const sourceHash = 'corporate_ca_sha256="$(sha256sum "$compact_ca_bundle" | cut -d \' \' -f 1)"'; + const corporateCaBuildArg = '--build-arg "NEMOCLAW_CORPORATE_CA_B64=${corporate_ca_b64}"'; const installedHash = "installed_ca_sha256=\"$( docker run --rm --network none --entrypoint sha256sum nemoclaw-hermes-plan-boundary /usr/local/share/nemoclaw/corporate-ca.pem | cut -d ' ' -f 1 )\""; const matchingHash = 'test "$installed_ca_sha256" = "$corporate_ca_sha256"'; @@ -688,6 +689,7 @@ function validateHermesMessagingPlanCaFixture( compactEncoding, sourceHash, "check-messaging-plan-image-boundary.mts plan", + corporateCaBuildArg, "check-production-build-args.sh", 'docker build "${build_args[@]}" -t nemoclaw-hermes-plan-boundary', installedHash, From 91cbdf8d8fb16d1e04d2ecb9fc07d93bb46189ee Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 14:45:45 -0700 Subject: [PATCH 15/19] fix(ci): document CA output file identity check Signed-off-by: Carlos Villela --- scripts/checks/select-ci-endpoint-ca-roots.mts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/checks/select-ci-endpoint-ca-roots.mts b/scripts/checks/select-ci-endpoint-ca-roots.mts index f278d2e400d..81ee7bf7ede 100644 --- a/scripts/checks/select-ci-endpoint-ca-roots.mts +++ b/scripts/checks/select-ci-endpoint-ca-roots.mts @@ -227,6 +227,7 @@ export function writeCiEndpointCaRootsOutput(outputPath: string, bundle: string) let fd: number; try { + // lgtm[js/file-system-race] Pre-open and post-open device and inode checks bind the descriptor to the validated path. fd = fs.openSync( outputPath, fs.constants.O_WRONLY | noFollow | (fs.constants.O_NONBLOCK ?? 0), From 6ef59fd1033c89defe65b99e2300b0c2131b383d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 15:08:33 -0700 Subject: [PATCH 16/19] fix(security): validate opened CA output before writes Signed-off-by: Carlos Villela --- .../checks/select-ci-endpoint-ca-roots.mts | 9 +-------- test/select-ci-endpoint-ca-roots.test.ts | 20 ++++++++++++------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/scripts/checks/select-ci-endpoint-ca-roots.mts b/scripts/checks/select-ci-endpoint-ca-roots.mts index 81ee7bf7ede..cf503d47ddb 100644 --- a/scripts/checks/select-ci-endpoint-ca-roots.mts +++ b/scripts/checks/select-ci-endpoint-ca-roots.mts @@ -220,14 +220,9 @@ export function writeCiEndpointCaRootsOutput(outputPath: string, bundle: string) throw new Error("output requires O_NOFOLLOW support"); } - const beforeOpen = fs.lstatSync(outputPath); - if (!beforeOpen.isFile() || beforeOpen.isSymbolicLink() || beforeOpen.nlink !== 1) { - throw new Error("output must be an existing regular file with exactly one link"); - } - let fd: number; try { - // lgtm[js/file-system-race] Pre-open and post-open device and inode checks bind the descriptor to the validated path. + // Open without following symlinks or blocking on special files, then validate before writing. fd = fs.openSync( outputPath, fs.constants.O_WRONLY | noFollow | (fs.constants.O_NONBLOCK ?? 0), @@ -246,8 +241,6 @@ export function writeCiEndpointCaRootsOutput(outputPath: string, bundle: string) !afterOpen.isFile() || afterOpen.isSymbolicLink() || afterOpen.nlink !== 1 || - beforeOpen.dev !== opened.dev || - beforeOpen.ino !== opened.ino || opened.dev !== afterOpen.dev || opened.ino !== afterOpen.ino ) { diff --git a/test/select-ci-endpoint-ca-roots.test.ts b/test/select-ci-endpoint-ca-roots.test.ts index da45f3ec69c..37e9363583d 100644 --- a/test/select-ci-endpoint-ca-roots.test.ts +++ b/test/select-ci-endpoint-ca-roots.test.ts @@ -219,7 +219,7 @@ describe("CI endpoint CA root selection", () => { writeCiEndpointCaRootsOutput(output, "replacement"); - expect(calls).toEqual(["lstat", "open", "fstat", "lstat", "truncate"]); + expect(calls).toEqual(["open", "fstat", "lstat", "truncate"]); expect(fs.readFileSync(output, "utf8")).toBe("replacement"); expect(fs.statSync(output).mode & 0o777).toBe(0o600); }); @@ -229,12 +229,14 @@ describe("CI endpoint CA root selection", () => { const created = spawnSync("mkfifo", [output], { encoding: "utf8", timeout: 5_000 }); expect(created.status, created.stderr).toBe(0); const openSync = vi.spyOn(fs, "openSync"); + const ftruncateSync = vi.spyOn(fs, "ftruncateSync"); expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( - "output must be an existing regular file with exactly one link", + "output must be an existing regular file that is not a symlink", ); - expect(openSync).not.toHaveBeenCalled(); + expect(openSync).toHaveBeenCalledOnce(); + expect(ftruncateSync).not.toHaveBeenCalled(); expect(fs.lstatSync(output).isFIFO()).toBe(true); }); @@ -245,12 +247,14 @@ describe("CI endpoint CA root selection", () => { fs.writeFileSync(target, "target", { mode: 0o640 }); fs.symlinkSync(target, output); const openSync = vi.spyOn(fs, "openSync"); + const ftruncateSync = vi.spyOn(fs, "ftruncateSync"); expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( - "output must be an existing regular file with exactly one link", + "output must be an existing regular file that is not a symlink", ); - expect(openSync).not.toHaveBeenCalled(); + expect(openSync).toHaveBeenCalledOnce(); + expect(ftruncateSync).not.toHaveBeenCalled(); expect(fs.lstatSync(output).isSymbolicLink()).toBe(true); expect(fs.readFileSync(target, "utf8")).toBe("target"); expect(fs.statSync(target).mode & 0o777).toBe(0o640); @@ -265,12 +269,14 @@ describe("CI endpoint CA root selection", () => { fs.writeFileSync(output, "unchanged", { mode: 0o640 }); fs.linkSync(output, linked); const openSync = vi.spyOn(fs, "openSync"); + const ftruncateSync = vi.spyOn(fs, "ftruncateSync"); expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( - "output must be an existing regular file with exactly one link", + "output must remain the same regular file with exactly one link", ); - expect(openSync).not.toHaveBeenCalled(); + expect(openSync).toHaveBeenCalledOnce(); + expect(ftruncateSync).not.toHaveBeenCalled(); for (const file of [output, linked]) { expect(fs.readFileSync(file, "utf8")).toBe("unchanged"); expect(fs.statSync(file).mode & 0o777).toBe(0o640); From 8bcb8f885ae984e2ef8bd0a0653ede1e07e76985 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 4 Aug 2026 15:30:28 -0700 Subject: [PATCH 17/19] test(security): cover device CA outputs Signed-off-by: Carlos Villela --- test/select-ci-endpoint-ca-roots.test.ts | 56 ++++++++++++++++-------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/test/select-ci-endpoint-ca-roots.test.ts b/test/select-ci-endpoint-ca-roots.test.ts index 37e9363583d..e30a207094e 100644 --- a/test/select-ci-endpoint-ca-roots.test.ts +++ b/test/select-ci-endpoint-ca-roots.test.ts @@ -224,7 +224,7 @@ describe("CI endpoint CA root selection", () => { expect(fs.statSync(output).mode & 0o777).toBe(0o600); }); - it.skipIf(process.platform === "win32")("rejects a FIFO CA output before opening it", () => { + it.skipIf(process.platform === "win32")("rejects a FIFO CA output without truncating it", () => { const output = path.join(tmpDir(), "compact.pem"); const created = spawnSync("mkfifo", [output], { encoding: "utf8", timeout: 5_000 }); expect(created.status, created.stderr).toBe(0); @@ -240,25 +240,45 @@ describe("CI endpoint CA root selection", () => { expect(fs.lstatSync(output).isFIFO()).toBe(true); }); - it.skipIf(process.platform === "win32")("rejects a symlinked CA output before opening it", () => { - const directory = tmpDir(); - const target = path.join(directory, "target.pem"); - const output = path.join(directory, "compact.pem"); - fs.writeFileSync(target, "target", { mode: 0o640 }); - fs.symlinkSync(target, output); - const openSync = vi.spyOn(fs, "openSync"); - const ftruncateSync = vi.spyOn(fs, "ftruncateSync"); + it.skipIf(process.platform === "win32")( + "rejects a symlinked CA output without opening its target", + () => { + const directory = tmpDir(); + const target = path.join(directory, "target.pem"); + const output = path.join(directory, "compact.pem"); + fs.writeFileSync(target, "target", { mode: 0o640 }); + fs.symlinkSync(target, output); + const openSync = vi.spyOn(fs, "openSync"); + const ftruncateSync = vi.spyOn(fs, "ftruncateSync"); - expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( - "output must be an existing regular file that is not a symlink", - ); + expect(() => writeCiEndpointCaRootsOutput(output, "replacement")).toThrow( + "output must be an existing regular file that is not a symlink", + ); - expect(openSync).toHaveBeenCalledOnce(); - expect(ftruncateSync).not.toHaveBeenCalled(); - expect(fs.lstatSync(output).isSymbolicLink()).toBe(true); - expect(fs.readFileSync(target, "utf8")).toBe("target"); - expect(fs.statSync(target).mode & 0o777).toBe(0o640); - }); + expect(openSync).toHaveBeenCalledOnce(); + expect(ftruncateSync).not.toHaveBeenCalled(); + expect(fs.lstatSync(output).isSymbolicLink()).toBe(true); + expect(fs.readFileSync(target, "utf8")).toBe("target"); + expect(fs.statSync(target).mode & 0o777).toBe(0o640); + }, + ); + + it.skipIf(process.platform === "win32")( + "rejects a device CA output without truncating it", + () => { + const ftruncateSync = vi.spyOn(fs, "ftruncateSync"); + const writeFileSync = vi.spyOn(fs, "writeFileSync"); + const fchmodSync = vi.spyOn(fs, "fchmodSync"); + + expect(() => writeCiEndpointCaRootsOutput("/dev/null", "replacement")).toThrow( + "output must remain the same regular file with exactly one link", + ); + + expect(ftruncateSync).not.toHaveBeenCalled(); + expect(writeFileSync).not.toHaveBeenCalled(); + expect(fchmodSync).not.toHaveBeenCalled(); + }, + ); it.skipIf(process.platform === "win32")( "rejects a hard-linked CA output without changing either path", From e2d6b0451591f07ac31fd983f05311bd1b906b34 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 4 Aug 2026 21:13:09 -0700 Subject: [PATCH 18/19] ci(gates): refresh pull request gate state From 33b741c46dc7ce8d2c9978218e1c0be9d52e46f9 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 4 Aug 2026 21:14:14 -0700 Subject: [PATCH 19/19] ci(gates): bind exact review receipts