diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 9538cf26f64..0431ae9901a 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -37,6 +37,9 @@ on: - "nemoclaw-blueprint/blueprint.yaml" - "scripts/lib/openclaw-npm-remediation.mts" - "scripts/lib/reviewed-npm-audit.mts" + - "scripts/security/build-native-security-packages.sh" + - "scripts/security/patches/libssh2-1.11.1-cve-2026.patch" + - "scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch" - "scripts/checks/node-tar-image-scan.mts" - "scripts/patch-bundled-npm-brace-expansion.mts" - "scripts/patch-bundled-npm-tar.mts" diff --git a/Dockerfile b/Dockerfile index 9ad859406ad..237677a0489 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1736,6 +1736,8 @@ RUN set -eu; \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ | cmp -s - "$security_inventory"; \ test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1"; \ @@ -1743,10 +1745,18 @@ RUN set -eu; \ test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1"; \ test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1"; \ + test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1"; \ + test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1"; \ ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5'; \ test "$(jq --version)" = "jq-1.8.2"; \ printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null; \ python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; \ + printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c -; \ + python3 -c "import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')"; \ + python3 -c "import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')"; \ vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 '; \ test -z "$(dpkg --audit)" # End completed-image security package verification. diff --git a/Dockerfile.base b/Dockerfile.base index d5630581681..a4210ee6bde 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -56,6 +56,28 @@ ARG PERL_VERSION=5.44.0 ARG PERL_SHA256=505cf43912e9480495c344c70260452e32aa2a73c546a026b3f100053b23ce91 ARG PERL_PACKAGE_REVISION=1nemoclaw1 +# Debian trixie has not published libssh2 or Python packages containing the +# reviewed upstream fixes. Build native packages once and install the same +# artifacts into the managed OpenClaw image. +FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba AS native-security-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.12 \ + ca-certificates=20250419 \ + curl=8.14.1-2+deb13u4 \ + git=1:2.47.3-0+deb13u1 \ + libssl-dev=3.5.6-1~deb13u2 \ + openssh-server=1:10.0p1-7+deb13u4 \ + xz-utils=5.8.1-1+deb13u1 \ + zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh +COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch +COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch + +RUN bash /scripts/security/build-native-security-packages.sh /out + # Debian trixie has not published a Perl package containing the upstream # fixes for CVE-2026-12087, CVE-2026-13221, and CVE-2026-57433. Build the # fixed upstream release as native Debian packages so dpkg dependencies and @@ -66,6 +88,8 @@ ARG PERL_VERSION ARG PERL_SHA256 ARG PERL_PACKAGE_REVISION +COPY --from=native-security-builder /out /tmp/security-packages + RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential=12.12 \ ca-certificates=20250419 \ @@ -157,11 +181,15 @@ RUN package_version="${PERL_VERSION}-${PERL_PACKAGE_REVISION}" \ 'Maintainer: NVIDIA NemoClaw Maintainers' \ 'Description: Perl 5 language interpreter metapackage for the NemoClaw sandbox' \ > /tmp/perl-meta/DEBIAN/control \ - && dpkg-deb --build --root-owner-group /tmp/perl-root /tmp/perl-base.deb \ - && dpkg-deb --build --root-owner-group /tmp/perl-meta /tmp/perl.deb + && dpkg-deb --build --root-owner-group \ + /tmp/perl-root /tmp/security-packages/perl-base.deb \ + && dpkg-deb --build --root-owner-group \ + /tmp/perl-meta /tmp/security-packages/perl.deb FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba +COPY --from=perl-builder /tmp/security-packages /tmp/nemoclaw-native-security + # OpenShell blocks the link-local EC2 Instance Metadata Service. Keep AWS SDK # credential chains from attempting an impossible metadata discovery path. ENV AWS_EC2_METADATA_DISABLED=true @@ -254,17 +282,27 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "$security_deb_dir/jq.deb" \ "$security_deb_dir/vim-common.deb" \ "$security_deb_dir/vim-tiny.deb" \ + /tmp/nemoclaw-native-security/libssh2-1t64.deb \ + /tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb \ && test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1" \ && test "$(dpkg-query -W -f='${Version}' libjq1)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1" \ && test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1" \ + && test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1" \ + && test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1" \ && test "$(dpkg-query -W -f='${Version}' perl)" = "$perl_version_before" \ && test "$(jq --version)" = "jq-1.8.2" \ && ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5' \ && printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null \ && python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION" \ + && printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c - \ + && python3 -c "import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')" \ + && python3 -c "import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')" \ && vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 ' \ && install -d -o root -g root -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' \ @@ -275,6 +313,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ > /usr/local/share/nemoclaw/security-packages.txt \ && chown root:root /usr/local/share/nemoclaw/security-packages.txt \ && chmod 0444 /usr/local/share/nemoclaw/security-packages.txt \ @@ -282,11 +322,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && ln -s /usr/bin/python3 /usr/local/bin/python -COPY --from=perl-builder /tmp/perl-base.deb /tmp/perl.deb /tmp/ - RUN apt-get update \ - && apt-get install -y --no-install-recommends /tmp/perl-base.deb /tmp/perl.deb \ - && rm -f /tmp/perl-base.deb /tmp/perl.deb \ + && apt-get install -y --no-install-recommends \ + /tmp/nemoclaw-native-security/perl-base.deb \ + /tmp/nemoclaw-native-security/perl.deb \ + && rm -rf /tmp/nemoclaw-native-security \ && rm -rf /var/lib/apt/lists/* \ && test "$(perl -e 'print $^V')" = "v5.44.0" \ && test "$(perl -MSocket -e 'print Socket->VERSION')" = "2.041" \ diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 324034c5deb..12edfe44507 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -6,7 +6,7 @@ # Layers PR-specific code (plugin, config, startup script) on top of the # pre-built Hermes base image. Mirrors the OpenClaw Dockerfile structure. -ARG BASE_IMAGE=ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:c925afe7c0742474166a4813a59f5961a93da4d130e9b35fb04088a384702142 +ARG BASE_IMAGE=ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:77975858f6d737cb5e46fb30e99acfccf9a912b9ab7bf022b652540a8de2db33 ARG NEMOCLAW_CORPORATE_CA_B64= FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba AS mcp-tool-discovery-runtime @@ -768,6 +768,8 @@ RUN set -eu; \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ | cmp -s - "$security_inventory"; \ test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1"; \ @@ -775,10 +777,18 @@ RUN set -eu; \ test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1"; \ test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1"; \ + test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1"; \ + test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1"; \ ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5'; \ test "$(jq --version)" = "jq-1.8.2"; \ printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null; \ python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; \ + printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c -; \ + python3 -c "import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')"; \ + python3 -c "import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')"; \ vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 '; \ test -z "$(dpkg --audit)" # End completed-image security package verification. diff --git a/agents/hermes/Dockerfile.base b/agents/hermes/Dockerfile.base index 40d5a083115..21d4a369fd8 100644 --- a/agents/hermes/Dockerfile.base +++ b/agents/hermes/Dockerfile.base @@ -19,8 +19,29 @@ # 6. New .hermes subdirectory — add mkdir/chmod below # ──────────────────────────────────────────────────────────────── +FROM node:24-trixie-slim@sha256:05c08ce4291e9a58f59456a7985176defb12cdd42271f35ff81a3e167ea61d4c AS native-security-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.12 \ + ca-certificates=20250419 \ + curl=8.14.1-2+deb13u4 \ + git=1:2.47.3-0+deb13u1 \ + libssl-dev=3.5.6-1~deb13u2 \ + openssh-server=1:10.0p1-7+deb13u4 \ + xz-utils=5.8.1-1+deb13u1 \ + zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh +COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch +COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch + +RUN bash /scripts/security/build-native-security-packages.sh /out + FROM node:24-trixie-slim@sha256:05c08ce4291e9a58f59456a7985176defb12cdd42271f35ff81a3e167ea61d4c +COPY --from=native-security-builder /out /tmp/nemoclaw-native-security + ENV DEBIAN_FRONTEND=noninteractive # Hermes version pinned for reproducibility. All four values below are @@ -122,16 +143,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "$security_deb_dir/jq.deb" \ "$security_deb_dir/vim-common.deb" \ "$security_deb_dir/vim-tiny.deb" \ + /tmp/nemoclaw-native-security/libssh2-1t64.deb \ + /tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb \ && test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1" \ && test "$(dpkg-query -W -f='${Version}' libjq1)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1" \ && test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1" \ + && test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1" \ + && test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1" \ && test "$(jq --version)" = "jq-1.8.2" \ && ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5' \ && printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null \ && python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION" \ + && printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c - \ + && python3 -c "import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')" \ + && python3 -c "import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')" \ && vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 ' \ && install -d -o root -g root -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' \ @@ -142,9 +173,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ > /usr/local/share/nemoclaw/security-packages.txt \ && chown root:root /usr/local/share/nemoclaw/security-packages.txt \ && chmod 0444 /usr/local/share/nemoclaw/security-packages.txt \ + && rm -rf /tmp/nemoclaw-native-security \ && rm -rf "$security_deb_dir" \ && rm -rf /var/lib/apt/lists/* diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index 84453d1cad0..73147ad3c92 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -237,6 +237,8 @@ RUN set -eu; \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ | cmp -s - "$security_inventory"; \ test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1"; \ @@ -244,10 +246,18 @@ RUN set -eu; \ test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1"; \ test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1"; \ + test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1"; \ + test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1"; \ ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5'; \ test "$(jq --version)" = "jq-1.8.2"; \ printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null; \ python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; \ + printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c -; \ + python3 -c "import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')"; \ + python3 -c "import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')"; \ vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 '; \ test -z "$(dpkg --audit)" # End completed-image security package verification. diff --git a/agents/langchain-deepagents-code/Dockerfile.base b/agents/langchain-deepagents-code/Dockerfile.base index 63e5f9449c4..8f9407498e3 100644 --- a/agents/langchain-deepagents-code/Dockerfile.base +++ b/agents/langchain-deepagents-code/Dockerfile.base @@ -7,8 +7,29 @@ # Node for NemoClaw build-time config generation, Python, shell tools, and a # hash-locked deepagents-code install with the NVIDIA provider extra. +FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba AS native-security-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.12 \ + ca-certificates=20250419 \ + curl=8.14.1-2+deb13u4 \ + git=1:2.47.3-0+deb13u1 \ + libssl-dev=3.5.6-1~deb13u2 \ + openssh-server=1:10.0p1-7+deb13u4 \ + xz-utils=5.8.1-1+deb13u1 \ + zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh +COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch +COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch + +RUN bash /scripts/security/build-native-security-packages.sh /out + FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba +COPY --from=native-security-builder /out /tmp/nemoclaw-native-security + COPY scripts/lib/reviewed-npm-archive.mts /scripts/lib/reviewed-npm-archive.mts COPY scripts/patch-bundled-npm-brace-expansion.mts /scripts/patch-bundled-npm-brace-expansion.mts COPY scripts/patch-bundled-npm-tar.mts /scripts/patch-bundled-npm-tar.mts @@ -97,16 +118,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "$security_deb_dir/jq.deb" \ "$security_deb_dir/vim-common.deb" \ "$security_deb_dir/vim-tiny.deb" \ + /tmp/nemoclaw-native-security/libssh2-1t64.deb \ + /tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb \ && test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1" \ && test "$(dpkg-query -W -f='${Version}' libjq1)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1" \ && test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1" \ + && test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1" \ + && test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1" \ && test "$(jq --version)" = "jq-1.8.2" \ && ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5' \ && printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null \ && python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION" \ + && printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c - \ + && python3 -c "import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')" \ + && python3 -c "import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')" \ && vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 ' \ && install -d -o root -g root -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' \ @@ -117,9 +148,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ > /usr/local/share/nemoclaw/security-packages.txt \ && chown root:root /usr/local/share/nemoclaw/security-packages.txt \ && chmod 0444 /usr/local/share/nemoclaw/security-packages.txt \ + && rm -rf /tmp/nemoclaw-native-security \ && rm -rf "$security_deb_dir" \ && rm -rf /var/lib/apt/lists/* diff --git a/ci/full-e2e-cold-path-calibration.json b/ci/full-e2e-cold-path-calibration.json index 5f638ae2aa9..0c6652bf53a 100644 --- a/ci/full-e2e-cold-path-calibration.json +++ b/ci/full-e2e-cold-path-calibration.json @@ -1,5 +1,5 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nFive independent workflow_dispatch full-e2e samples of current-main baseline 1a74b8348c2182fbf806726341186f008444e28e with the phase-measurement changes at 4544d07c8bfd500c3b64a74380ef5cd0e62089f5. All runs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn. Each baseline budget is derived independently so phase caps diagnose regressions; they are not portions that must sum to the root-start budget. The separate validation adjustment records four existing full-e2e job observations from three descendant heads after f62c278bd737f4f47be2e85436f65b270d5b4280 added the reviewed WeChat runtime graph to the supported sandbox image. The relevant image-building inputs listed in imageInputPaths were unchanged through 2adc8481ff3053a5a7be37d130cb183e222934ff; repository tests enforce that ancestry and unchanged-input boundary. All four jobs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn; two exceeded the prior root-start and/or sandbox-phase caps. The conclusion fields record the full-e2e job conclusions, not aggregate workflow conclusions. The adjustment raises only those two caps from the maximum observed value plus the existing headroom, rounded up to one second. This is a bounded post-change validation adjustment, not a replacement five-run single-SHA calibration. Retire it by replacing the baseline and removing validationAdjustment after five successful full-e2e samples from one SHA that contains the image change. The authoritative-local-build adjustment records two functional same-head PR runs that emitted the required local base-build reason. Its allowance is the maximum excess over the normal root-start or sandbox-phase budget plus an operator-selected stability floor of 27 seconds (or 10 percent, if larger), rounded up to one second. It applies only when that reason is observed and does not alter published-image budgets.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nFive independent workflow_dispatch full-e2e samples of current-main baseline 1a74b8348c2182fbf806726341186f008444e28e with the phase-measurement changes at 4544d07c8bfd500c3b64a74380ef5cd0e62089f5. All runs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn. Each baseline budget is derived independently so phase caps diagnose regressions; they are not portions that must sum to the root-start budget. The separate validation adjustment records four existing full-e2e job observations from three descendant heads after f62c278bd737f4f47be2e85436f65b270d5b4280 added the reviewed WeChat runtime graph to the supported sandbox image. The relevant image-building inputs listed in imageInputPaths were unchanged through 2adc8481ff3053a5a7be37d130cb183e222934ff; repository tests enforce that ancestry and unchanged-input boundary. All four jobs completed install, BuildKit prebuild without fallback, the silence assertion, and the expected first turn; two exceeded the prior root-start and/or sandbox-phase caps. The conclusion fields record the full-e2e job conclusions, not aggregate workflow conclusions. The adjustment raises only those two caps from the maximum observed value plus the existing headroom, rounded up to one second. This is a bounded post-change validation adjustment, not a replacement five-run single-SHA calibration. Retire it by replacing the baseline and removing validationAdjustment after five successful full-e2e samples from one SHA that contains the image change. The authoritative-local-build adjustment records functional PR runs that emitted the required local base-build reason. The latest observation covers the longer source build required when reviewed native security packages change. Its allowance is the maximum excess over the normal root-start or sandbox-phase budget plus an operator-selected stability floor of 27 seconds (or 10 percent, if larger), rounded up to one second. It applies only when that reason is observed and does not alter published-image budgets.", "schemaVersion": 1, "calibratedAt": "2026-07-13", "baselineMainSha": "1a74b8348c2182fbf806726341186f008444e28e", @@ -328,7 +328,7 @@ } }, "authoritativeLocalBaseBuildAdjustment": { - "validatedAt": "2026-07-21", + "validatedAt": "2026-07-28", "triggerOutput": "Building OpenClaw sandbox base image locally because no compatible published base image was found.", "adjustedMetrics": [ "rootStartToFirstTurnCompletion", @@ -345,6 +345,12 @@ "runId": 29871968836, "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29871968836", "headSha": "188d9a75b3e5efdafeb38e885138bb196197574f", + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stderr.txt", + "output": "Building OpenClaw sandbox base image locally because no compatible published base image was found." + }, + "nativeSecurityInputPaths": [], "rootStartToFirstTurnCompletionMs": 277769, "sandboxPhaseMs": 225927 }, @@ -352,11 +358,42 @@ "runId": 29872724335, "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/29872724335", "headSha": "188d9a75b3e5efdafeb38e885138bb196197574f", + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stderr.txt", + "output": "Building OpenClaw sandbox base image locally because no compatible published base image was found." + }, + "nativeSecurityInputPaths": [], "rootStartToFirstTurnCompletionMs": 281762, "sandboxPhaseMs": 233584 + }, + { + "runId": 30375843683, + "runUrl": "https://github.com/NVIDIA/NemoClaw/actions/runs/30375843683", + "headSha": "5f190e4948a11f8b05655e085c11803ce0a9a0a8", + "triggerEvidence": { + "artifact": "e2e-full-e2e", + "path": "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stderr.txt", + "output": "Building OpenClaw sandbox base image locally because no compatible published base image was found." + }, + "nativeSecurityInputPaths": [ + "Dockerfile.base", + "scripts/security/build-native-security-packages.sh", + "scripts/security/patches/libssh2-1.11.1-cve-2026.patch", + "scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch" + ], + "rootStartToFirstTurnCompletionMs": 778744, + "sandboxPhaseMs": 725351 } ], - "derivedAllowanceMs": 90000 + "retirement": { + "trigger": "successful-single-sha-calibration", + "minimumSampleCount": 5, + "allSamplesSameHead": true, + "nativeSecurityInputsMustBeUnchanged": true, + "action": "replace-baseline-and-remove-adjustment" + }, + "derivedAllowanceMs": 610000 }, "derivedBudgetsMs": { "rootStartToFirstTurnCompletionBudgetMs": 205000, diff --git a/ci/onboard-performance-budget.json b/ci/onboard-performance-budget.json index cdfeb1968fa..979deb25d37 100644 --- a/ci/onboard-performance-budget.json +++ b/ci/onboard-performance-budget.json @@ -1,5 +1,5 @@ { - "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. Four later functional full-e2e jobs completed install, BuildKit prebuild without fallback, and the expected first turn on their listed heads after the supported sandbox image changed; their separately recorded bounded validation adjustment raises only the root-start and sandbox-phase caps to the observed maximum plus 10 percent, rounded up to one second. The adjustment is retired after five successful samples from one head containing the image change, when that calibration replaces the baseline and the adjustment is removed. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from the same-head PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets.", + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nInitial advisory budget for the cloud-onboard-e2e warm-system trace signal. Profiling traces from #3769 were not available in durable CI artifacts when #3776 was implemented. The latest three release tags only exposed one tag-matching trace artifact, from a failed v0.0.66 nightly run, so this cap is calibrated from the latest ten distinct successful main full-trace samples available on 2026-06-23. Those samples had total durations of 298250 ms, 296926 ms, 304190 ms, 294859 ms, 305013 ms, 316147 ms, 300843 ms, 292702 ms, 201332 ms, and 206250 ms; the cap uses p95 via linear interpolation (index 8.55 between samples 8 and 9) plus 25 percent, rounded up to the nearest 30 seconds.\n\nThe full-e2e cold-path baseline budgets are derived from the five current-main samples recorded in ci/full-e2e-cold-path-calibration.json. For each interval and phase independently, the baseline is nearest-rank p95 plus the larger of 5 seconds or 10 percent, rounded up to the nearest second. Four later functional full-e2e jobs completed install, BuildKit prebuild without fallback, and the expected first turn on their listed heads after the supported sandbox image changed; their separately recorded bounded validation adjustment raises only the root-start and sandbox-phase caps to the observed maximum plus 10 percent, rounded up to one second. The adjustment is retired after five successful samples from one head containing the image change, when that calibration replaces the baseline and the adjustment is removed. Independent phase caps are diagnostic regression gates, not additive portions of the root-start interval. The authoritative local base-build allowance is derived separately from exact PR evidence recorded in ci/full-e2e-cold-path-calibration.json. It applies only when full-e2e observes the exact local-build reason and adjusts only the root-start and sandbox-phase caps; published-image runs retain the normal budgets.", "schemaVersion": 1, "mode": "advisory", "scope": "cloud-onboard-e2e warm-system", @@ -13,7 +13,7 @@ "minPercent": 30 }, "fullE2eColdPath": { - "authoritativeLocalBaseBuildAllowanceMs": 90000, + "authoritativeLocalBaseBuildAllowanceMs": 610000, "rootStartToFirstTurnCompletionBudgetMs": 228000, "rootEndToFirstTurnCompletionBudgetMs": 14000, "phaseBudgetsMs": { diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 969a13b0438..4ec7d785416 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -358,7 +358,7 @@ }, { "file": "test/onboard-performance-config-schema.test.ts", - "test": "keeps the authoritative local-build allowance tied to same-head PR evidence", + "test": "keeps the authoritative local-build allowance tied to exact PR evidence", "category": "compatibility" }, { diff --git a/docs/security/sandbox-base-2026-07-27-native-package-review.md b/docs/security/sandbox-base-2026-07-27-native-package-review.md new file mode 100644 index 00000000000..8afb8c87dc6 --- /dev/null +++ b/docs/security/sandbox-base-2026-07-27-native-package-review.md @@ -0,0 +1,98 @@ + + + +# Sandbox base native package review: libssh2 and Python HTMLParser + +Date: 2026-07-27 + +## Scope + +This review covers two temporary native-package remediations shared by the OpenClaw, Hermes, and Deep Agents Code base images: + +- backport the reviewed upstream fixes for CVE-2026-66032, CVE-2026-66033, CVE-2026-66034, and CVE-2026-66035 to libssh2 1.11.1; and +- backport the reviewed CPython 3.13 fix for CVE-2026-15308 to Debian's Python 3.13.5 standard library. + +The supported Debian suite has not published packages containing these fixes. +The changes preserve the existing image behavior and do not create a new integration or product surface. + +## Reviewed identities + +| Component | Input identity | Fixed identity | Upstream fix | +| --- | --- | --- | --- | +| libssh2 | `libssh2-1t64=1.11.1-1+deb13u1` | `libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1` | `5e4776146552d898b9c0e1b313cd093fa8dc92d0`, `a2ed82d40964bbc0d64cd717aa0a5a892117d2e6`, `a13bb6c773f0d55ad1628cede57e99803cd898d9`, and `42e33d81577ed4b95d4b4f6f845e5ee8efe5eeb4` | +| Python HTMLParser | `libpython3.13-stdlib=3.13.5-2+deb13u4` | `nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1` | `7933f4bf7131aa4140750f9404f5de0aa2969ced` | + +The libssh2 source archive is bound to SHA-256 `9954cb54c4f548198a7cbebad248bdc87dd64bd26185708a294b2b50771e3769`. +The original Debian packages are downloaded from the immutable `20260724T000000Z` Debian snapshot. +The original Debian package SHA-256 values are: + +| Package | amd64 | arm64 | +| --- | --- | --- | +| `libssh2-1t64_1.11.1-1+deb13u1` | `915c4ec450a369d430e0151f9e10e25044ea2f0d6e41901e00a9317e232e5683` | `600c2a845d6d14d292c765382bc7e644898762e1634a4aecf5b85329622dbbfe` | +| `libpython3.13-stdlib_3.13.5-2+deb13u4` | `0def2d972310b59704ad119abee5a97f95409e14ff1359edd8cc7b8892cfd43f` | `37cce6086b7c1ca93086f83b68761737607689e634693b6972b5dbfd6c080872` | + +The unmodified Debian `html/parser.py` input is bound to SHA-256 `f91ec3de6331206bbe2ec3e54a05f646bd23d3c61a18d4a01b25164e070bacc9`. +The fixed file is bound to SHA-256 `4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7`. +The file contains the reviewed upstream batching fix plus an empty-input guard so repeated `feed("")` calls remain no-ops instead of accumulating pending entries. + +## Package contracts + +The shared builder applies the reviewed patches only after verifying the source artifacts. +It builds libssh2 with OpenSSL and zlib support and runs all 43 upstream test cases against a local OpenSSH fixture. +The builder disables only libssh2's nested Docker orchestration, which is unavailable during an image build, and executes those same cases against the fixture directly. +It then verifies the `libssh2.so.1` soname and rejects a build that removes any exported symbol from the original Debian library. +It then replaces only the shared library in the original Debian runtime package and records the NemoClaw package revision. + +The Python fix package owns only `/usr/lib/python3.13/html/parser.py`. +It depends on the exact Debian `libpython3.13-stdlib=3.13.5-2+deb13u4` input and declares the narrow file replacement for that version. +This avoids changing Python's exact internal package dependency chain while making the fixed file visible as a separate dpkg identity. + +Every managed base image: + +1. installs both native packages; +2. verifies their exact dpkg versions; +3. verifies the fixed HTMLParser file hash; +4. verifies that `html.parser` resolves to the exact hashed file; +5. verifies 20,000 empty `feed()` calls leave no pending entries and exercises incremental HTML comment parsing across 20,000 input chunks; +6. loads `libssh2.so.1` and verifies the 1.11.1 runtime identity; +7. records both packages in the root-owned, read-only security inventory; and +8. removes the temporary package artifacts. + +The completed production images repeat the inventory, package, file, and runtime checks and require an empty `dpkg --audit` result. + +## Concern ledger + +### SEC-1: libssh2 1.11.1 lacks four upstream memory-safety fixes + +- Surface: native SSH2 client library +- Severity: high +- Confidence: high +- Failure mode: malformed SSH, SFTP, or public-key data can reach missing bounds or lifetime checks. +- Disposition: backport, build, test, package, runtime-proof +- Validation: native amd64 and arm64 image builds passed for all managed images. + +### SEC-2: Python 3.13.5 HTMLParser repeatedly rescans incomplete input + +- Surface: Python standard library +- Severity: high +- Confidence: high +- Failure mode: repeated small `feed()` calls with an unterminated construct can cause quadratic CPU consumption. +- Disposition: backport, package, file-hash-proof, behavior-test +- Validation: native amd64 and arm64 image builds passed for all managed images. + +## Removal conditions + +Remove the libssh2 backport after the supported Debian suite publishes a package containing all four reviewed commits and that package passes the same symbol, upstream test, runtime, and multi-architecture image checks. + +Remove the Python file replacement after the supported Debian suite publishes a Python 3.13 package containing commit `7933f4bf7131aa4140750f9404f5de0aa2969ced` or its reviewed successor and the replacement passes the same incremental-parser and multi-architecture image checks. + +## Verification + +Required evidence for the final pull-request head: + +- shared-builder syntax and immutable-input contract tests; +- exact base-package and completed-image contract tests for all three managed images on amd64 and arm64; +- the full libssh2 upstream test suite in each native builder; +- symbol and soname compatibility against the original Debian libssh2 package; +- repository formatting and type checks; and +- successful native base-image builds for all managed images. diff --git a/scripts/security/build-native-security-packages.sh b/scripts/security/build-native-security-packages.sh new file mode 100755 index 00000000000..8c7537c3d27 --- /dev/null +++ b/scripts/security/build-native-security-packages.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly LIBSSH2_VERSION="1.11.1" +readonly LIBSSH2_DEBIAN_VERSION="1.11.1-1+deb13u1" +readonly LIBSSH2_PACKAGE_VERSION="${LIBSSH2_DEBIAN_VERSION}+nemoclaw1" +readonly LIBSSH2_SOURCE_SHA256="9954cb54c4f548198a7cbebad248bdc87dd64bd26185708a294b2b50771e3769" +readonly PYTHON_DEBIAN_VERSION="3.13.5-2+deb13u4" +readonly PYTHON_FIX_VERSION="${PYTHON_DEBIAN_VERSION}+nemoclaw1" +readonly PYTHON_PARSER_SHA256="f91ec3de6331206bbe2ec3e54a05f646bd23d3c61a18d4a01b25164e070bacc9" +readonly PYTHON_PARSER_FIXED_SHA256="4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" +readonly DEBIAN_SNAPSHOT_URL="https://snapshot.debian.org/archive/debian/20260724T000000Z/pool/main" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly script_dir +readonly patch_dir="${script_dir}/patches" +readonly output_dir="${1:-/out}" +build_root="$(mktemp -d /tmp/nemoclaw-native-security.XXXXXX)" +readonly build_root + +cleanup() { + rm -rf "${build_root}" +} +trap cleanup EXIT + +download() { + local url="$1" + local output="$2" + curl --proto '=https' --tlsv1.2 -fsSL \ + --retry 5 --retry-all-errors --retry-delay 2 \ + --connect-timeout 15 --max-time 120 \ + -o "${output}" "${url}" +} + +verify_sha256() { + local expected="$1" + local path="$2" + printf '%s %s\n' "${expected}" "${path}" | sha256sum -c - +} + +rewrite_control_field() { + local control="$1" + local field="$2" + local value="$3" + sed -i "s/^${field}: .*$/${field}: ${value}/" "${control}" + grep -Fqx "${field}: ${value}" "${control}" +} + +refresh_md5sums() { + local package_root="$1" + ( + cd "${package_root}" + find . -path ./DEBIAN -prune -o -type f -printf '%P\0' \ + | sort -z \ + | xargs -0 md5sum + ) >"${package_root}/DEBIAN/md5sums" +} + +read_make_list() { + local makefile="$1" + local variable="$2" + + awk -v variable="${variable}" ' + $0 ~ "^" variable "[[:space:]]*=" { + capture = 1 + sub("^[^=]*=[[:space:]]*", "") + } + capture { + continued = sub(/[[:space:]]*\\[[:space:]]*$/, "") + for (field = 1; field <= NF; field++) { + print "./" $field + } + if (!continued) { + exit + } + } + ' "${makefile}" +} + +run_libssh2_tests() { + local source_dir="$1" + local test_output + local test_user="libssh2" + local -a docker_tests + local -a sshd_tests + local -a full_tests + + mapfile -t docker_tests < <( + read_make_list "${source_dir}/tests/Makefile.inc" DOCKER_TESTS + ) + mapfile -t sshd_tests < <( + read_make_list "${source_dir}/tests/Makefile.inc" SSHD_TESTS + ) + test "${#docker_tests[@]}" -eq 22 + test "${#sshd_tests[@]}" -eq 2 + test "$(wc -l <"${source_dir}/tests/test_read_algos.txt")" -eq 18 + full_tests=( + "${docker_tests[@]}" + "${sshd_tests[@]}" + ./test_read_algos.test + ) + + # sshd reads AuthorizedKeysFile after dropping privileges to the fixture + # user, so the mktemp parent must be traversable during the test run. + chmod o+x "$(dirname -- "${source_dir}")" + if ! id "${test_user}" >/dev/null 2>&1; then + useradd --create-home --shell /bin/bash "${test_user}" + fi + printf '%s\n' "${test_user}:my test password" | chpasswd + install -d -o "${test_user}" -g "${test_user}" \ + "/home/${test_user}/.ssh" \ + "/home/${test_user}/sandbox" + install -o "${test_user}" -g "${test_user}" -m 0600 \ + "${source_dir}/tests/openssh_server/authorized_keys" \ + "/home/${test_user}/.ssh/authorized_keys" + sed -i \ + 's/session[[:space:]]*required[[:space:]]*pam_loginuid.so/session optional pam_loginuid.so/' \ + /etc/pam.d/sshd + + ( + cd "${source_dir}" + make check + ) + + if ! test_output="$( + cd "${source_dir}/tests" + USER="${test_user}" \ + LOGNAME="${test_user}" \ + SSHD_FLAGS="-o UsePAM=yes -o KbdInteractiveAuthentication=yes -o PasswordAuthentication=yes -o PerSourcePenalties=no" \ + ./test_sshd.test "${full_tests[@]}" 2>&1 + )"; then + printf '%s\n' "${test_output}" >&2 + return 1 + fi + printf '%s\n' "${test_output}" + if grep -Eq '^not ok([[:space:]]|$)' <<<"${test_output}"; then + printf 'A nested libssh2 TAP test reported a failure.\n' >&2 + return 1 + fi +} + +build_libssh2_package() { + local architecture="$1" + local original_sha256 + local original_deb="${build_root}/libssh2-original.deb" + local source_archive="${build_root}/libssh2.tar.xz" + local source_dir="${build_root}/libssh2-source" + local install_root="${build_root}/libssh2-install" + local package_root="${build_root}/libssh2-package" + local multiarch + + case "${architecture}" in + amd64) + original_sha256="915c4ec450a369d430e0151f9e10e25044ea2f0d6e41901e00a9317e232e5683" + ;; + arm64) + original_sha256="600c2a845d6d14d292c765382bc7e644898762e1634a4aecf5b85329622dbbfe" + ;; + *) + printf 'Unsupported architecture: %s\n' "${architecture}" >&2 + return 64 + ;; + esac + + download \ + "https://libssh2.org/download/libssh2-${LIBSSH2_VERSION}.tar.xz" \ + "${source_archive}" + verify_sha256 "${LIBSSH2_SOURCE_SHA256}" "${source_archive}" + mkdir -p "${source_dir}" + tar -xJf "${source_archive}" -C "${source_dir}" --strip-components=1 + git -C "${source_dir}" apply --check \ + "${patch_dir}/libssh2-1.11.1-cve-2026.patch" + git -C "${source_dir}" apply \ + "${patch_dir}/libssh2-1.11.1-cve-2026.patch" + + ( + cd "${source_dir}" + ./configure \ + --prefix=/usr \ + --disable-static \ + --disable-docker-tests \ + --disable-sshd-tests \ + --with-crypto=openssl \ + --with-libz + make -j"$(nproc)" + run_libssh2_tests "${source_dir}" + make DESTDIR="${install_root}" install + ) + + multiarch="$(gcc -print-multiarch)" + test -n "${multiarch}" + test -f "${install_root}/usr/lib/libssh2.so.1.0.1" + + download \ + "${DEBIAN_SNAPSHOT_URL}/libs/libssh2/libssh2-1t64_${LIBSSH2_DEBIAN_VERSION}_${architecture}.deb" \ + "${original_deb}" + verify_sha256 "${original_sha256}" "${original_deb}" + dpkg-deb -R "${original_deb}" "${package_root}" + test "$(dpkg-deb -f "${original_deb}" Package)" = "libssh2-1t64" + test "$(dpkg-deb -f "${original_deb}" Version)" = "${LIBSSH2_DEBIAN_VERSION}" + + local original_library="${package_root}/usr/lib/${multiarch}/libssh2.so.1.0.1" + local fixed_library="${install_root}/usr/lib/libssh2.so.1.0.1" + test -f "${original_library}" + readelf -d "${fixed_library}" | grep -Fq '(SONAME)' \ + && readelf -d "${fixed_library}" | grep -Fq '[libssh2.so.1]' + nm -D --defined-only --format=posix "${original_library}" \ + | cut -d ' ' -f 1 | sort -u >"${build_root}/libssh2-original.symbols" + nm -D --defined-only --format=posix "${fixed_library}" \ + | cut -d ' ' -f 1 | sort -u >"${build_root}/libssh2-fixed.symbols" + comm -23 \ + "${build_root}/libssh2-original.symbols" \ + "${build_root}/libssh2-fixed.symbols" \ + >"${build_root}/libssh2-missing.symbols" + test ! -s "${build_root}/libssh2-missing.symbols" + + install -m 0644 "${fixed_library}" "${original_library}" + rewrite_control_field \ + "${package_root}/DEBIAN/control" Version "${LIBSSH2_PACKAGE_VERSION}" + rewrite_control_field \ + "${package_root}/DEBIAN/control" Provides \ + "libssh2-1 (= ${LIBSSH2_PACKAGE_VERSION})" + rewrite_control_field \ + "${package_root}/DEBIAN/control" Breaks \ + "libssh2-1 (<< ${LIBSSH2_PACKAGE_VERSION})" + refresh_md5sums "${package_root}" + dpkg-deb --build --root-owner-group \ + "${package_root}" \ + "${output_dir}/libssh2-1t64.deb" + test "$(dpkg-deb -f "${output_dir}/libssh2-1t64.deb" Version)" = \ + "${LIBSSH2_PACKAGE_VERSION}" +} + +build_python_fix_package() { + local architecture="$1" + local original_sha256 + local original_deb="${build_root}/python-stdlib-original.deb" + local original_root="${build_root}/python-stdlib-original" + local package_root="${build_root}/python-htmlparser-fix" + local parser_path="usr/lib/python3.13/html/parser.py" + + case "${architecture}" in + amd64) + original_sha256="0def2d972310b59704ad119abee5a97f95409e14ff1359edd8cc7b8892cfd43f" + ;; + arm64) + original_sha256="37cce6086b7c1ca93086f83b68761737607689e634693b6972b5dbfd6c080872" + ;; + *) + printf 'Unsupported architecture: %s\n' "${architecture}" >&2 + return 64 + ;; + esac + + download \ + "${DEBIAN_SNAPSHOT_URL}/p/python3.13/libpython3.13-stdlib_${PYTHON_DEBIAN_VERSION}_${architecture}.deb" \ + "${original_deb}" + verify_sha256 "${original_sha256}" "${original_deb}" + dpkg-deb -x "${original_deb}" "${original_root}" + test "$(dpkg-deb -f "${original_deb}" Package)" = "libpython3.13-stdlib" + test "$(dpkg-deb -f "${original_deb}" Version)" = "${PYTHON_DEBIAN_VERSION}" + verify_sha256 "${PYTHON_PARSER_SHA256}" "${original_root}/${parser_path}" + + mkdir -p "${package_root}/DEBIAN" \ + "$(dirname -- "${package_root}/${parser_path}")" + install -m 0644 \ + "${original_root}/${parser_path}" \ + "${package_root}/${parser_path}" + git -C "${package_root}" apply --check \ + "${patch_dir}/python3.13-htmlparser-cve-2026-15308.patch" + git -C "${package_root}" apply \ + "${patch_dir}/python3.13-htmlparser-cve-2026-15308.patch" + verify_sha256 \ + "${PYTHON_PARSER_FIXED_SHA256}" \ + "${package_root}/${parser_path}" + + printf '%s\n' \ + 'Package: nemoclaw-python3.13-htmlparser-fix' \ + "Version: ${PYTHON_FIX_VERSION}" \ + 'Architecture: all' \ + 'Priority: optional' \ + 'Section: python' \ + "Depends: libpython3.13-stdlib (= ${PYTHON_DEBIAN_VERSION})" \ + "Replaces: libpython3.13-stdlib (<= ${PYTHON_DEBIAN_VERSION})" \ + 'Maintainer: NVIDIA NemoClaw Maintainers' \ + 'Description: NemoClaw HTMLParser security backport for Python 3.13' \ + ' Backports the upstream fix for incremental parsing complexity.' \ + >"${package_root}/DEBIAN/control" + refresh_md5sums "${package_root}" + dpkg-deb --build --root-owner-group \ + "${package_root}" \ + "${output_dir}/nemoclaw-python3.13-htmlparser-fix.deb" + test "$(dpkg-deb -f "${output_dir}/nemoclaw-python3.13-htmlparser-fix.deb" Version)" = \ + "${PYTHON_FIX_VERSION}" +} + +main() { + local architecture + + mkdir -p "${output_dir}" + architecture="$(dpkg --print-architecture)" + build_libssh2_package "${architecture}" + build_python_fix_package "${architecture}" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main +fi diff --git a/scripts/security/patches/libssh2-1.11.1-cve-2026.patch b/scripts/security/patches/libssh2-1.11.1-cve-2026.patch new file mode 100644 index 00000000000..b9d49149ca7 --- /dev/null +++ b/scripts/security/patches/libssh2-1.11.1-cve-2026.patch @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Backports the following verified libssh2 upstream commits to 1.11.1: +# 5e4776146552d898b9c0e1b313cd093fa8dc92d0 +# a2ed82d40964bbc0d64cd717aa0a5a892117d2e6 +# a13bb6c773f0d55ad1628cede57e99803cd898d9 +# 42e33d81577ed4b95d4b4f6f845e5ee8efe5eeb4 +diff --git a/src/openssl.c b/src/openssl.c +index 9af96cb..fb1b28b 100644 +--- a/src/openssl.c ++++ b/src/openssl.c +@@ -1041,11 +1041,13 @@ _libssh2_cipher_crypt(_libssh2_cipher_ctx * ctx, + const int aadlen = (is_aesgcm && IS_FIRST(firstlast)) ? 4 : 0; + /* size of AT, if present */ + const int authenticationtag = IS_LAST(firstlast) ? authlen : 0; +- /* length to encrypt */ +- const int cryptlen = (unsigned int)blocksize - aadlen - authenticationtag; ++ unsigned int cryptlen; /* length to encrypt */ + + (void)algo; + +- assert(blocksize <= sizeof(buf)); +- assert(cryptlen >= 0); ++ if(blocksize > sizeof(buf) || ++ blocksize < (size_t)(aadlen + authenticationtag)) ++ return 1; ++ ++ cryptlen = (unsigned int)blocksize - aadlen - authenticationtag; + + #if LIBSSH2_AES_GCM + /* First block */ +diff --git a/src/publickey.c b/src/publickey.c +index 8517e5a..9a5825f 100644 +--- a/src/publickey.c ++++ b/src/publickey.c +@@ -988,6 +988,13 @@ libssh2_publickey_list_fetch(LIBSSH2_PUBLICKEY *pkey, + } + + if(comment_len) { ++ if(pkey->listFetch_s + comment_len > ++ pkey->listFetch_data + pkey->listFetch_data_len) { ++ _libssh2_error(session, LIBSSH2_ERROR_BUFFER_TOO_SMALL, ++ "ListFetch data too short"); ++ goto err_exit; ++ } ++ + list[keys].num_attrs = 1; + list[keys].attrs = + LIBSSH2_ALLOC(session, +diff --git a/src/sftp.c b/src/sftp.c +index 43f2f93..4c80f18 100644 +--- a/src/sftp.c ++++ b/src/sftp.c +@@ -1278,6 +1278,7 @@ static LIBSSH2_SFTP_HANDLE *sftp_open(LIBSSH2_SFTP *sftp, + "got HANDLE FXOK")); + + LIBSSH2_FREE(session, data); ++ data = NULL; + + /* silly situation, but check for a HANDLE */ + rc = sftp_packet_require(sftp, SSH_FXP_HANDLE, +diff --git a/src/transport.c b/src/transport.c +index b16531b..89f1cf3 100644 +--- a/src/transport.c ++++ b/src/transport.c +@@ -235,13 +235,19 @@ static int transport_fullpacket(LIBSSH2_SESSION *session, + unsigned char *decrypt_buffer; + int blocksize = session->remote.crypt->blocksize; + ++ if(p->total_num < mac_len + 4 + (size_t)blocksize) { ++ LIBSSH2_FREE(session, p->payload); ++ p->payload = NULL; ++ return LIBSSH2_ERROR_DECRYPT; ++ } ++ decrypt_size = (ssize_t)(p->total_num - mac_len - 4); ++ + rc = decrypt(session, p->payload + 4, + first_block, blocksize, FIRST_BLOCK); + if(rc) { + return rc; + } + + /* we need buffer for decrypt */ +- decrypt_size = p->total_num - mac_len - 4; + decrypt_buffer = LIBSSH2_ALLOC(session, decrypt_size); + if(!decrypt_buffer) { + return LIBSSH2_ERROR_ALLOC; diff --git a/scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch b/scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch new file mode 100644 index 00000000000..eab13bee010 --- /dev/null +++ b/scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Backports verified CPython 3.13 commit +# 7933f4bf7131aa4140750f9404f5de0aa2969ced to Debian 3.13.5 and treats an +# empty feed as a no-op so zero-length calls cannot grow the pending buffer. +diff --git a/usr/lib/python3.13/html/parser.py b/usr/lib/python3.13/html/parser.py +index 84a32e8..f25b948 100644 +--- a/usr/lib/python3.13/html/parser.py ++++ b/usr/lib/python3.13/html/parser.py +@@ -115,6 +115,9 @@ class HTMLParser(_markupbase.ParserBase): + self.lasttag = '???' + self.interesting = interesting_normal + self.cdata_elem = None ++ self._pending = [] ++ self._pending_len = 0 ++ self._parse_threshold = 1 + super().reset() + + def feed(self, data): +@@ -123,6 +126,29 @@ class HTMLParser(_markupbase.ParserBase): + Call this as often as you want, with as little or as much text + as you want (may include '\n'). + """ +- self.rawdata = self.rawdata + data +- self.goahead(0) ++ # Accumulate new data in a list and only join and parse it once ++ # enough has piled up. Rescanning an unparsed buffer (e.g. an ++ # unterminated tag) and concatenating onto it on every call would ++ # both be quadratic in the input size. ++ if not data: ++ return ++ self._pending_len += len(data) ++ if self._pending_len < self._parse_threshold: ++ self._pending.append(data) ++ else: ++ if not self._pending: ++ self.rawdata += data ++ else: ++ self._pending.append(data) ++ self.rawdata += ''.join(self._pending) ++ self._pending.clear() ++ self._pending_len = 0 ++ n = len(self.rawdata) ++ self.goahead(0) ++ if len(self.rawdata) < n: ++ # Some data was parsed; resume on the next call. ++ self._parse_threshold = 1 ++ else: ++ # Nothing was parsed; wait until the buffer doubles. ++ self._parse_threshold = len(self.rawdata) + +@@ -129,5 +155,9 @@ class HTMLParser(_markupbase.ParserBase): + def close(self): + """Handle any buffered data.""" ++ if self._pending: ++ self.rawdata += ''.join(self._pending) ++ self._pending.clear() ++ self._pending_len = 0 + self.goahead(1) + + __starttag_text = None diff --git a/src/commands/sandbox/mcp.test.ts b/src/commands/sandbox/mcp.test.ts new file mode 100644 index 00000000000..f8cb020057e --- /dev/null +++ b/src/commands/sandbox/mcp.test.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + dispatchMcpBridgeCommand: vi.fn().mockResolvedValue(undefined), + moduleLoaded: vi.fn(), +})); + +vi.mock("../../lib/actions/sandbox/mcp-bridge", () => { + mocks.moduleLoaded(); + return { + dispatchMcpBridgeCommand: mocks.dispatchMcpBridgeCommand, + }; +}); + +import SandboxMcpCommand from "./mcp"; + +const rootDir = process.cwd(); + +describe("sandbox MCP oclif command", () => { + it("loads the MCP lifecycle only when command execution reaches dispatch", async () => { + expect(mocks.moduleLoaded).not.toHaveBeenCalled(); + + await SandboxMcpCommand.run(["alpha", "list"], rootDir); + + expect(mocks.moduleLoaded).toHaveBeenCalledOnce(); + expect(mocks.dispatchMcpBridgeCommand).toHaveBeenCalledWith("alpha", ["list"]); + }); +}); diff --git a/src/commands/sandbox/mcp.ts b/src/commands/sandbox/mcp.ts index 14464f945f2..c531cae658b 100644 --- a/src/commands/sandbox/mcp.ts +++ b/src/commands/sandbox/mcp.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dispatchMcpBridgeCommand } from "../../lib/actions/sandbox/mcp-bridge"; import { NemoClawCommand } from "../../lib/cli/nemoclaw-oclif-command"; export default class SandboxMcpCommand extends NemoClawCommand { @@ -33,6 +32,7 @@ export default class SandboxMcpCommand extends NemoClawCommand { ); return; } + const { dispatchMcpBridgeCommand } = await import("../../lib/actions/sandbox/mcp-bridge"); await dispatchMcpBridgeCommand(sandboxName, actionArgs); } } diff --git a/src/lib/onboard/base-image.ts b/src/lib/onboard/base-image.ts index b562a5c643e..8234e59336c 100644 --- a/src/lib/onboard/base-image.ts +++ b/src/lib/onboard/base-image.ts @@ -43,7 +43,7 @@ export function openClawBaseImageHasSecurityInventory(imageRef: string): boolean 'test -f "$security_inventory"', 'test ! -L "$security_inventory"', `test "$(stat -c '%u:%g:%a' "$security_inventory")" = "0:0:444"`, - `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" | cmp -s - "$security_inventory"`, + `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" | cmp -s - "$security_inventory"`, `printf '%s\\n' "${OPENCLAW_SECURITY_INVENTORY_PROBE_OK}"`, ].join("; "), ], diff --git a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts index 447e75e1c2b..f63bef91cbd 100644 --- a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts +++ b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts @@ -54,6 +54,9 @@ const CANONICAL_POST_GENERATOR_RUN_SHA256 = new Set([ "1197b99bdb996b37a3e4e386a507dfabcdfb2c26a40b015d617f97208668187d", "5cc53ef9c588470f325c5df8189a2eb1525140d947332ae2c0a80fccb2f36ccb", "83567d1fa0e73bef6a3333383c13ace05e26704964ae6a7a76ee24a2f2be3d7e", + "ca1f7b1cb9dd5d467f806792c4072a84ef1e6402c3e8650b6325b95cc186ccdf", + "7e6a6879382f833f17be02ca7d287685b6afa1c423b1e087b3b05dd677d6e325", + "22406cef76f7a66a3d527c17f2a5bc6a217c71c753c979406fb2c3fa7cd8f0eb", ]); function instructionSha256(text: string): string { diff --git a/test/fixtures/security/python3.13-html-parser-fixed.txt b/test/fixtures/security/python3.13-html-parser-fixed.txt new file mode 100644 index 00000000000..faf38754f50 --- /dev/null +++ b/test/fixtures/security/python3.13-html-parser-fixed.txt @@ -0,0 +1,523 @@ +"""A parser for HTML and XHTML.""" + +# This file is based on sgmllib.py, but the API is slightly different. + +# XXX There should be a way to distinguish between PCDATA (parsed +# character data -- the normal case), RCDATA (replaceable character +# data -- only char and entity references and end tags are special) +# and CDATA (character data -- only end tags are special). + + +import re +import _markupbase + +from html import unescape +from html.entities import html5 as html5_entities + + +__all__ = ['HTMLParser'] + +# Regular expressions used for parsing + +interesting_normal = re.compile('[&<]') +incomplete = re.compile('&[a-zA-Z#]') + +entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') +charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') + +starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('') +commentclose = re.compile(r'--\s*>') +# Note: +# 1) if you change tagfind/attrfind remember to update locatestarttagend too; +# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# explode, so don't do it. +# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state +# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') +attrfind_tolerant = re.compile( + r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +locatestarttagend_tolerant = re.compile(r""" + <[a-zA-Z][^\t\n\r\f />\x00]* # tag name + (?:[\s/]* # optional whitespace before attribute name + (?:(?<=['"\s/])[^\s/>][^\s/=>]* # attribute name + (?:\s*=+\s* # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\s]* # bare value + ) + \s* # possibly followed by a space + )?(?:\s|/(?!>))* + )* + )? + \s* # trailing whitespace +""", re.VERBOSE) +endendtag = re.compile('>') +# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between +# ') + +# Character reference processing logic specific to attribute values +# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state +def _replace_attr_charref(match): + ref = match.group(0) + # Numeric / hex char refs must always be unescaped + if ref.startswith('&#'): + return unescape(ref) + # Named character / entity references must only be unescaped + # if they are an exact match, and they are not followed by an equals sign + if not ref.endswith('=') and ref[1:] in html5_entities: + return unescape(ref) + # Otherwise do not unescape + return ref + +def _unescape_attrvalue(s): + return attr_charref.sub(_replace_attr_charref, s) + + +class HTMLParser(_markupbase.ParserBase): + """Find tags and other markup and call handler functions. + + Usage: + p = HTMLParser() + p.feed(data) + ... + p.close() + + Start tags are handled by calling self.handle_starttag() or + self.handle_startendtag(); end tags by self.handle_endtag(). The + data between tags is passed from the parser to the derived class + by calling self.handle_data() with the data as argument (the data + may be split up in arbitrary chunks). If convert_charrefs is + True the character references are converted automatically to the + corresponding Unicode character (and self.handle_data() is no + longer split in chunks), otherwise they are passed by calling + self.handle_entityref() or self.handle_charref() with the string + containing respectively the named or numeric reference as the + argument. + """ + + CDATA_CONTENT_ELEMENTS = ("script", "style") + + def __init__(self, *, convert_charrefs=True): + """Initialize and reset this instance. + + If convert_charrefs is True (the default), all character references + are automatically converted to the corresponding Unicode characters. + """ + super().__init__() + self.convert_charrefs = convert_charrefs + self.reset() + + def reset(self): + """Reset this instance. Loses all unprocessed data.""" + self.rawdata = '' + self.lasttag = '???' + self.interesting = interesting_normal + self.cdata_elem = None + self._pending = [] + self._pending_len = 0 + self._parse_threshold = 1 + super().reset() + + def feed(self, data): + r"""Feed data to the parser. + + Call this as often as you want, with as little or as much text + as you want (may include '\n'). + """ + # Accumulate new data in a list and only join and parse it once + # enough has piled up. Rescanning an unparsed buffer (e.g. an + # unterminated tag) and concatenating onto it on every call would + # both be quadratic in the input size. + if not data: + return + self._pending_len += len(data) + if self._pending_len < self._parse_threshold: + self._pending.append(data) + else: + if not self._pending: + self.rawdata += data + else: + self._pending.append(data) + self.rawdata += ''.join(self._pending) + self._pending.clear() + self._pending_len = 0 + n = len(self.rawdata) + self.goahead(0) + if len(self.rawdata) < n: + # Some data was parsed; resume on the next call. + self._parse_threshold = 1 + else: + # Nothing was parsed; wait until the buffer doubles. + self._parse_threshold = len(self.rawdata) + + def close(self): + """Handle any buffered data.""" + if self._pending: + self.rawdata += ''.join(self._pending) + self._pending.clear() + self._pending_len = 0 + self.goahead(1) + + __starttag_text = None + + def get_starttag_text(self): + """Return full source of start tag: '<...>'.""" + return self.__starttag_text + + def set_cdata_mode(self, elem): + self.cdata_elem = elem.lower() + self.interesting = re.compile(r'' % self.cdata_elem, re.I) + + def clear_cdata_mode(self): + self.interesting = interesting_normal + self.cdata_elem = None + + # Internal -- handle data as far as reasonable. May leave state + # and data to be processed by a subsequent call. If 'end' is + # true, force handling all data as if followed by EOF marker. + def goahead(self, end): + rawdata = self.rawdata + i = 0 + n = len(rawdata) + while i < n: + if self.convert_charrefs and not self.cdata_elem: + j = rawdata.find('<', i) + if j < 0: + # if we can't find the next <, either we are at the end + # or there's more text incoming. If the latter is True, + # we can't pass the text to handle_data in case we have + # a charref cut in half at end. Try to determine if + # this is the case before proceeding by looking for an + # & near the end and see if it's followed by a space or ;. + amppos = rawdata.rfind('&', max(i, n-34)) + if (amppos >= 0 and + not re.compile(r'[\s;]').search(rawdata, amppos)): + break # wait till we get all the text + j = n + else: + match = self.interesting.search(rawdata, i) # < or & + if match: + j = match.start() + else: + if self.cdata_elem: + break + j = n + if i < j: + if self.convert_charrefs and not self.cdata_elem: + self.handle_data(unescape(rawdata[i:j])) + else: + self.handle_data(rawdata[i:j]) + i = self.updatepos(i, j) + if i == n: break + startswith = rawdata.startswith + if startswith('<', i): + if starttagopen.match(rawdata, i): # < + letter + k = self.parse_starttag(i) + elif startswith(" + gtpos = rawdata.find('>', i+9) + if gtpos == -1: + return -1 + self.handle_decl(rawdata[i+2:gtpos]) + return gtpos+1 + else: + return self.parse_bogus_comment(i) + + # Internal -- parse bogus comment, return length or -1 if not terminated + # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state + def parse_bogus_comment(self, i, report=1): + rawdata = self.rawdata + assert rawdata[i:i+2] in ('', i+2) + if pos == -1: + return -1 + if report: + self.handle_comment(rawdata[i+2:pos]) + return pos + 1 + + # Internal -- parse processing instr, return end or -1 if not terminated + def parse_pi(self, i): + rawdata = self.rawdata + assert rawdata[i:i+2] == ' + if not match: + return -1 + j = match.start() + self.handle_pi(rawdata[i+2: j]) + j = match.end() + return j + + # Internal -- handle starttag, return end or -1 if not terminated + def parse_starttag(self, i): + self.__starttag_text = None + endpos = self.check_for_whole_start_tag(i) + if endpos < 0: + return endpos + rawdata = self.rawdata + self.__starttag_text = rawdata[i:endpos] + + # Now parse the data between i+1 and j into a tag and attrs + attrs = [] + match = tagfind_tolerant.match(rawdata, i+1) + assert match, 'unexpected call to parse_starttag()' + k = match.end() + self.lasttag = tag = match.group(1).lower() + while k < endpos: + m = attrfind_tolerant.match(rawdata, k) + if not m: + break + attrname, rest, attrvalue = m.group(1, 2, 3) + if not rest: + attrvalue = None + elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ + attrvalue[:1] == '"' == attrvalue[-1:]: + attrvalue = attrvalue[1:-1] + if attrvalue: + attrvalue = _unescape_attrvalue(attrvalue) + attrs.append((attrname.lower(), attrvalue)) + k = m.end() + + end = rawdata[k:endpos].strip() + if end not in (">", "/>"): + self.handle_data(rawdata[i:endpos]) + return endpos + if end.endswith('/>'): + # XHTML-style empty tag: + self.handle_startendtag(tag, attrs) + else: + self.handle_starttag(tag, attrs) + if tag in self.CDATA_CONTENT_ELEMENTS: + self.set_cdata_mode(tag) + return endpos + + # Internal -- check to see if we have a complete starttag; return end + # or -1 if incomplete. + def check_for_whole_start_tag(self, i): + rawdata = self.rawdata + m = locatestarttagend_tolerant.match(rawdata, i) + if m: + j = m.end() + next = rawdata[j:j+1] + if next == ">": + return j + 1 + if next == "/": + if rawdata.startswith("/>", j): + return j + 2 + if rawdata.startswith("/", j): + # buffer boundary + return -1 + # else bogus input + if j > i: + return j + else: + return i + 1 + if next == "": + # end of input + return -1 + if next in ("abcdefghijklmnopqrstuvwxyz=/" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): + # end of input in or before attribute value, or we have the + # '/' from a '/>' ending + return -1 + if j > i: + return j + else: + return i + 1 + raise AssertionError("we should not get here!") + + # Internal -- parse endtag, return end or -1 if incomplete + def parse_endtag(self, i): + rawdata = self.rawdata + assert rawdata[i:i+2] == " + if not match: + return -1 + gtpos = match.end() + match = endtagfind.match(rawdata, i) # + if not match: + if self.cdata_elem is not None: + self.handle_data(rawdata[i:gtpos]) + return gtpos + # find the name: w3.org/TR/html5/tokenization.html#tag-name-state + namematch = tagfind_tolerant.match(rawdata, i+2) + if not namematch: + # w3.org/TR/html5/tokenization.html#end-tag-open-state + if rawdata[i:i+3] == '': + return i+3 + else: + return self.parse_bogus_comment(i) + tagname = namematch.group(1).lower() + # consume and ignore other stuff between the name and the > + # Note: this is not 100% correct, since we might have things like + # , but looking for > after the name should cover + # most of the cases and is much simpler + gtpos = rawdata.find('>', namematch.end()) + self.handle_endtag(tagname) + return gtpos+1 + + elem = match.group(1).lower() # script or style + if self.cdata_elem is not None: + if elem != self.cdata_elem: + self.handle_data(rawdata[i:gtpos]) + return gtpos + + self.handle_endtag(elem) + self.clear_cdata_mode() + return gtpos + + # Overridable -- finish processing of start+end tag: + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + self.handle_endtag(tag) + + # Overridable -- handle start tag + def handle_starttag(self, tag, attrs): + pass + + # Overridable -- handle end tag + def handle_endtag(self, tag): + pass + + # Overridable -- handle character reference + def handle_charref(self, name): + pass + + # Overridable -- handle entity reference + def handle_entityref(self, name): + pass + + # Overridable -- handle data + def handle_data(self, data): + pass + + # Overridable -- handle comment + def handle_comment(self, data): + pass + + # Overridable -- handle declaration + def handle_decl(self, decl): + pass + + # Overridable -- handle processing instruction + def handle_pi(self, data): + pass + + def unknown_decl(self, data): + pass diff --git a/test/helpers/base-apt-security-functions.ts b/test/helpers/base-apt-security-functions.ts index 11f8ea0fd8e..31c64485a77 100644 --- a/test/helpers/base-apt-security-functions.ts +++ b/test/helpers/base-apt-security-functions.ts @@ -50,7 +50,7 @@ export function runLoggedDockerShell(command: string, tmp: string, functionDefs: fs.writeFileSync(scriptPath, script, { mode: 0o700 }); return spawnSync("bash", [scriptPath], { encoding: "utf-8", - timeout: 5000, + timeout: 15000, }); } @@ -85,9 +85,9 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ` printf "${architecture}\\n"`, ' elif [[ "$#" -eq 1 && "$1" == "--audit" ]]; then', " return 0", - ' elif [[ "$#" -eq 7 && "$1" == "-i" && "${2##*/}" == "libexpat1.deb" && "${3##*/}" == "libonig5.deb" && "${4##*/}" == "libjq1.deb" && "${5##*/}" == "jq.deb" && "${6##*/}" == "vim-common.deb" && "${7##*/}" == "vim-tiny.deb" ]]; then', + ' elif [[ "$#" -eq 9 && "$1" == "-i" && "${2##*/}" == "libexpat1.deb" && "${3##*/}" == "libonig5.deb" && "${4##*/}" == "libjq1.deb" && "${5##*/}" == "jq.deb" && "${6##*/}" == "vim-common.deb" && "${7##*/}" == "vim-tiny.deb" && "${8##*/}" == "libssh2-1t64.deb" && "${9##*/}" == "nemoclaw-python3.13-htmlparser-fix.deb" ]]; then', ' printf "dpkg-install\\n" >> "$call_log"', - ' [[ -f "$2" && -f "$3" && -f "$4" && -f "$5" && -f "$6" && -f "$7" ]]', + ' [[ -f "$2" && -f "$3" && -f "$4" && -f "$5" && -f "$6" && -f "$7" && -f "$8" && -f "$9" ]]', " else", " return 64", " fi", @@ -102,6 +102,8 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ' libjq1|jq) printf "1.8.2-1" ;;', ' perl) printf "5.40.1-6" ;;', ' vim-common|vim-tiny) printf "2:9.2.0782-1" ;;', + ' libssh2-1t64) printf "1.11.1-1+deb13u1+nemoclaw1" ;;', + ' nemoclaw-python3.13-htmlparser-fix) printf "3.13.5-2+deb13u4+nemoclaw1" ;;', " *) return 64 ;;", " esac", "}", @@ -127,7 +129,7 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri [ "sha256sum() {", ' [[ "$#" -eq 2 && "$1" == "-c" && "$2" == "-" ]] || return 64', - " local line path count=0", + " local line path count=0 parser_count=0", " while IFS= read -r line; do", ' path="${line#* }"', ' [[ -f "$path" ]] || return 1', @@ -138,11 +140,12 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ` "${hashes.jq} "*/jq.deb) ;;`, ' "6b063038246492c4a20e0a212c896dde4d5aa9f59d6fb43ff33d10080bc53a39 "*/vim-common.deb) ;;', ` "${hashes.vimTiny} "*/vim-tiny.deb) ;;`, + ' "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7 "*/python3.13/html/parser.py) (( parser_count += 1 )) ;;', " *) return 1 ;;", " esac", " (( count += 1 ))", " done", - ' [[ "$count" -eq 6 ]]', + ' [[ "$count" -eq 6 || ( "$count" -eq 1 && "$parser_count" -eq 1 ) ]]', "}", ].join("\n"), [ @@ -166,7 +169,13 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ].join("\n"), [ "python3() {", - ' [[ "$#" -eq 2 && "$1" == "-c" && "$2" == "import pyexpat; assert pyexpat.EXPAT_VERSION == \'expat_2.8.2\', pyexpat.EXPAT_VERSION" ]]', + ' [[ "$#" -eq 2 && "$1" == "-c" ]] || return 64', + ' case "$2" in', + " \"import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION\") ;;", + " \"import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')\") ;;", + " \"import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')\") ;;", + " *) return 64 ;;", + " esac", "}", ].join("\n"), [ diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 51b625dcba0..c84db62246a 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -71,7 +71,7 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [ 'test -f "$security_inventory"', 'test ! -L "$security_inventory"', `test "$(stat -c '%u:%g:%a' "$security_inventory")" = "0:0:444"`, - `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" | cmp -s - "$security_inventory"`, + `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" | cmp -s - "$security_inventory"`, `printf '%s\\n' "nemoclaw-security-inventory-ok"`, ].join("; "); diff --git a/test/helpers/python-parser-security-fixture.ts b/test/helpers/python-parser-security-fixture.ts new file mode 100644 index 00000000000..4965c0c2519 --- /dev/null +++ b/test/helpers/python-parser-security-fixture.ts @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +export const FIXED_PARSER_SHA256 = + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7"; + +export function htmlParserSecurityProbe(parserPath: string): string { + return `import sys; from pathlib import Path; import html.parser; Path(html.parser.__file__).resolve() == Path('${parserPath}').resolve() or sys.exit('html.parser loaded from an unexpected path'); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; p._pending == [] or sys.exit('empty feeds accumulated pending entries'); p.feed(''); p.close(); p.rawdata == '' or sys.exit('incremental parsing retained raw data')`; +} + +const PYEXPAT_PROBE = + "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; +const LIBSSH2_PROBE = + "import ctypes, sys; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')"; +const FIXED_PARSER_FIXTURE = path.resolve( + import.meta.dirname, + "..", + "fixtures", + "security", + "python3.13-html-parser-fixed.txt", +); + +export function stageFixedParser(tmp: string): { + fixedParser: string; + pythonShim: string; +} { + const parserBytes = fs.readFileSync(FIXED_PARSER_FIXTURE); + const parserHash = createHash("sha256").update(parserBytes).digest("hex"); + if (parserHash !== FIXED_PARSER_SHA256) { + throw new Error(`Fixed parser fixture hash mismatch: ${parserHash}`); + } + + const fixedParser = path.join(tmp, "python3.13", "html", "parser.py"); + const pythonShim = path.join(tmp, "usr-bin", "python3"); + fs.mkdirSync(path.dirname(fixedParser), { recursive: true }); + fs.mkdirSync(path.dirname(pythonShim), { recursive: true }); + fs.writeFileSync(fixedParser, parserBytes); + fs.writeFileSync( + pythonShim, + [ + "#!/usr/bin/env python3", + "import html", + "import importlib.util", + "from importlib.machinery import SourceFileLoader", + "import sys", + `parser_path = ${JSON.stringify(fixedParser)}`, + `html_probe = ${JSON.stringify(htmlParserSecurityProbe(fixedParser))}`, + `accepted_noop_probes = {${JSON.stringify(PYEXPAT_PROBE)}, ${JSON.stringify(LIBSSH2_PROBE)}}`, + "if len(sys.argv) != 3 or sys.argv[1] != '-c':", + " raise SystemExit(64)", + "code = sys.argv[2]", + "if code in accepted_noop_probes:", + " raise SystemExit(0)", + "if code != html_probe:", + " raise SystemExit(64)", + "loader = SourceFileLoader('html.parser', parser_path)", + "spec = importlib.util.spec_from_loader(loader.name, loader)", + "if spec is None:", + " raise SystemExit(65)", + "module = importlib.util.module_from_spec(spec)", + "sys.modules[loader.name] = module", + "loader.exec_module(module)", + "html.parser = module", + "exec(code, {})", + "", + ].join("\n"), + { mode: 0o755 }, + ); + return { fixedParser, pythonShim }; +} + +export function useRealPatchedParser(functionDefinitions: string[], pythonShim: string): string[] { + return [ + ...functionDefinitions.filter((definition) => !definition.startsWith("python3() {")), + `python3() { ${JSON.stringify(pythonShim)} "$@"; }`, + ]; +} diff --git a/test/hermes-share-mount-deps.test.ts b/test/hermes-share-mount-deps.test.ts index ec96be1af0a..db190e0c8a4 100644 --- a/test/hermes-share-mount-deps.test.ts +++ b/test/hermes-share-mount-deps.test.ts @@ -7,14 +7,21 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { BASE_APT_SECURITY_FUNCTIONS } from "./helpers/base-apt-security-functions"; +import { stageFixedParser, useRealPatchedParser } from "./helpers/python-parser-security-fixture"; const ROOT = path.resolve(import.meta.dirname, ".."); const HERMES_DOCKERFILE_BASE = path.join(ROOT, "agents", "hermes", "Dockerfile.base"); function extractAptInstallCommand(dockerfile: string): string { - const match = dockerfile.match( - /RUN\s+apt-get update\s*&&\s*apt-get install -y --no-install-recommends[\s\S]*?&&\s*rm -rf \/var\/lib\/apt\/lists\/\*/m, + const runtimeStage = dockerfile.lastIndexOf( + "FROM node:24-trixie-slim@sha256:05c08ce4291e9a58f59456a7985176defb12cdd42271f35ff81a3e167ea61d4c", ); + expect(runtimeStage).toBeGreaterThanOrEqual(0); + const match = dockerfile + .slice(runtimeStage) + .match( + /RUN\s+apt-get update\s*&&\s*apt-get install -y --no-install-recommends[\s\S]*?&&\s*rm -rf \/var\/lib\/apt\/lists\/\*/m, + ); expect(match).not.toBeNull(); return match![0].replace(/^RUN\s+/, "").replace(/\\\n/g, " "); } @@ -64,7 +71,7 @@ function runLoggedShell(command: string, tmp: string, functionDefs: string[] = [ command, ].join("\n"); fs.writeFileSync(scriptPath, script, { mode: 0o700 }); - const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000 }); + const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 15000 }); const calls = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf-8") : ""; return { result, calls }; } @@ -191,21 +198,32 @@ describe("Hermes share mount package parity (#2947)", () => { const dockerfile = fs.readFileSync(HERMES_DOCKERFILE_BASE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-share-apt-")); const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); + const debianSecurityDebs = path.join(tmp, "debian-security-debs"); + const nativeSecurityDebs = path.join(tmp, "native-security-debs"); const inventoryDirectory = path.join(tmp, "security-inventory"); const inventory = path.join(inventoryDirectory, "security-packages.txt"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); fs.mkdirSync(lists); + fs.mkdirSync(debianSecurityDebs); + fs.mkdirSync(nativeSecurityDebs); + fs.writeFileSync(path.join(nativeSecurityDebs, "libssh2-1t64.deb"), "fixed libssh2"); + fs.writeFileSync( + path.join(nativeSecurityDebs, "nemoclaw-python3.13-htmlparser-fix.deb"), + "fixed parser package", + ); try { const command = extractAptInstallCommand(dockerfile) .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) + .replaceAll("/tmp/nemoclaw-debian-security", debianSecurityDebs) + .replaceAll("/tmp/nemoclaw-native-security", nativeSecurityDebs) .replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory) - .replaceAll("/usr/local/share/nemoclaw", inventoryDirectory); + .replaceAll("/usr/local/share/nemoclaw", inventoryDirectory) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); const { result, calls } = runLoggedShell(command, tmp, [ 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...BASE_APT_SECURITY_FUNCTIONS, + ...useRealPatchedParser(BASE_APT_SECURITY_FUNCTIONS, pythonShim), ]); expect(result.status).toBe(0); @@ -214,6 +232,8 @@ describe("Hermes share mount package parity (#2947)", () => { expect(calls).toContain("procps=2:4.0.4-9"); expect(calls).toContain("e2fsprogs=1.47.2-3+b11"); expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u4"); + expect(fs.existsSync(debianSecurityDebs)).toBe(false); + expect(fs.existsSync(nativeSecurityDebs)).toBe(false); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/native-security-packages.test.ts b/test/native-security-packages.test.ts new file mode 100644 index 00000000000..ed5f82adb4a --- /dev/null +++ b/test/native-security-packages.test.ts @@ -0,0 +1,195 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const BUILD_SCRIPT = path.join(ROOT, "scripts", "security", "build-native-security-packages.sh"); +const LIBSSH2_PATCH = path.join( + ROOT, + "scripts", + "security", + "patches", + "libssh2-1.11.1-cve-2026.patch", +); +const PYTHON_PATCH = path.join( + ROOT, + "scripts", + "security", + "patches", + "python3.13-htmlparser-cve-2026-15308.patch", +); +const BASE_DOCKERFILES = [ + path.join(ROOT, "Dockerfile.base"), + path.join(ROOT, "agents", "hermes", "Dockerfile.base"), + path.join(ROOT, "agents", "langchain-deepagents-code", "Dockerfile.base"), +] as const; + +function runLibssh2Harness(nestedFailure = false) { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-libssh2-harness-")); + const testsDir = path.join(fixture, "tests"); + const harnessLog = path.join(fixture, "harness-log"); + fs.mkdirSync(path.join(testsDir, "openssh_server"), { recursive: true }); + fs.mkdirSync(harnessLog); + fs.writeFileSync( + path.join(testsDir, "Makefile.inc"), + [ + `DOCKER_TESTS = ${Array.from({ length: 22 }, (_, index) => `docker-${index + 1}`).join(" ")}`, + "SSHD_TESTS = sshd-1 sshd-2", + "", + ].join("\n"), + ); + fs.writeFileSync( + path.join(testsDir, "test_read_algos.txt"), + `${Array.from({ length: 18 }, (_, index) => `algorithm-${index + 1}`).join("\n")}\n`, + ); + fs.writeFileSync(path.join(testsDir, "openssh_server", "authorized_keys"), "fixture-key\n"); + fs.writeFileSync( + path.join(testsDir, "test_sshd.test"), + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'printf "%s\\n" "$USER|$LOGNAME|$SSHD_FLAGS" >"$HARNESS_LOG/environment"', + 'printf "%s\\n" "$@" >"$HARNESS_LOG/arguments"', + 'printf "1..25\\n"', + 'if [[ "${NESTED_FAILURE:-0}" == "1" ]]; then', + ' printf "not ok 7 - nested algorithm\\n"', + "else", + ' printf "ok 25 - all upstream cases\\n"', + "fi", + "", + ].join("\n"), + { mode: 0o700 }, + ); + + const result = spawnSync( + "bash", + [ + "-c", + [ + "set -euo pipefail", + 'source "$1"', + 'calls="$HARNESS_LOG/calls"', + 'mapfile() { local target="$2" line; eval "$target=()"; while IFS= read -r line; do eval "$target+=(\\"\\$line\\")"; done; }', + 'chmod() { printf "chmod %s\\n" "$*" >>"$calls"; }', + "id() { return 0; }", + 'useradd() { printf "useradd %s\\n" "$*" >>"$calls"; }', + 'chpasswd() { cat >/dev/null; printf "chpasswd\\n" >>"$calls"; }', + 'install() { printf "install %s\\n" "$*" >>"$calls"; }', + 'sed() { printf "sed %s\\n" "$*" >>"$calls"; }', + 'make() { printf "make %s\\n" "$*" >>"$calls"; }', + 'run_libssh2_tests "$2"', + ].join("\n"), + "libssh2-harness", + BUILD_SCRIPT, + fixture, + ], + { + encoding: "utf-8", + env: { + ...process.env, + HARNESS_LOG: harnessLog, + NESTED_FAILURE: nestedFailure ? "1" : "0", + }, + }, + ); + return { fixture, harnessLog, result }; +} + +describe("native security package remediation", () => { + it("keeps the package builder syntactically valid", () => { + const result = spawnSync("bash", ["-n", BUILD_SCRIPT], { encoding: "utf-8" }); + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); + }); + + it("runs every upstream libssh2 case against the local OpenSSH fixture", () => { + const { fixture, harnessLog, result } = runLibssh2Harness(); + try { + expect({ status: result.status, stderr: result.stderr }).toEqual({ + status: 0, + stderr: "", + }); + expect(result.stdout).toContain("ok 25 - all upstream cases"); + expect(fs.readFileSync(path.join(harnessLog, "calls"), "utf-8")).toContain("make check"); + const argumentsList = fs + .readFileSync(path.join(harnessLog, "arguments"), "utf-8") + .trim() + .split("\n"); + expect(argumentsList).toEqual([ + ...Array.from({ length: 22 }, (_, index) => `./docker-${index + 1}`), + "./sshd-1", + "./sshd-2", + "./test_read_algos.test", + ]); + expect(fs.readFileSync(path.join(harnessLog, "environment"), "utf-8").trim()).toBe( + "libssh2|libssh2|-o UsePAM=yes -o KbdInteractiveAuthentication=yes -o PasswordAuthentication=yes -o PerSourcePenalties=no", + ); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + + it("rejects a nested libssh2 TAP failure", () => { + const { fixture, result } = runLibssh2Harness(true); + try { + expect(result.status).toBe(1); + expect(result.stderr).toContain("A nested libssh2 TAP test reported a failure."); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } + }); + + it("records every reviewed upstream fix at the patch boundary", () => { + const libssh2Patch = fs.readFileSync(LIBSSH2_PATCH, "utf-8"); + for (const commit of [ + "5e4776146552d898b9c0e1b313cd093fa8dc92d0", + "a2ed82d40964bbc0d64cd717aa0a5a892117d2e6", + "a13bb6c773f0d55ad1628cede57e99803cd898d9", + "42e33d81577ed4b95d4b4f6f845e5ee8efe5eeb4", + ]) { + expect(libssh2Patch).toContain(commit); + } + expect(libssh2Patch).toContain("blocksize > sizeof(buf)"); + expect(libssh2Patch).toContain("pkey->listFetch_s + comment_len"); + expect(libssh2Patch).toContain("data = NULL"); + expect(libssh2Patch).toContain("p->total_num < mac_len + 4 + (size_t)blocksize"); + + const pythonPatch = fs.readFileSync(PYTHON_PATCH, "utf-8"); + expect(pythonPatch).toContain("7933f4bf7131aa4140750f9404f5de0aa2969ced"); + expect(pythonPatch).toContain("if not data:"); + expect(pythonPatch).toContain("self._pending_len += len(data)"); + expect(pythonPatch).toContain("self._parse_threshold = len(self.rawdata)"); + }); + + it.each(BASE_DOCKERFILES)("wires both native packages into %s", (dockerfile) => { + const content = fs.readFileSync(dockerfile, "utf-8"); + expect(content).toContain("AS native-security-builder"); + expect(content).toContain( + "COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh", + ); + expect(content).toContain( + "COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch", + ); + expect(content).toContain( + "COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch", + ); + expect(content).toContain("bash /scripts/security/build-native-security-packages.sh /out"); + expect(content).toContain("openssh-server=1:10.0p1-7+deb13u4"); + expect(content).toContain("/tmp/nemoclaw-native-security/libssh2-1t64.deb"); + expect(content).toContain( + "/tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb", + ); + expect(content).toContain("libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1"); + expect(content).toContain("nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1"); + expect(content).toContain("4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7"); + expect(content).toContain("[p.feed('') for _ in range(20000)]"); + expect(content).toContain("or sys.exit('empty feeds accumulated pending entries')"); + expect(content).toContain( + "lib.libssh2_version(0) == b'1.11.1' or sys.exit('unexpected libssh2 runtime version')", + ); + }); +}); diff --git a/test/onboard-performance-config-schema.test.ts b/test/onboard-performance-config-schema.test.ts index 27a1a291627..c9180f3ed70 100644 --- a/test/onboard-performance-config-schema.test.ts +++ b/test/onboard-performance-config-schema.test.ts @@ -99,9 +99,22 @@ interface Calibration { runId: number; runUrl: string; headSha: string; + triggerEvidence: { + artifact: string; + path: string; + output: string; + }; + nativeSecurityInputPaths: string[]; rootStartToFirstTurnCompletionMs: number; sandboxPhaseMs: number; }>; + retirement: { + trigger: string; + minimumSampleCount: number; + allSamplesSameHead: boolean; + nativeSecurityInputsMustBeUnchanged: boolean; + action: string; + }; derivedAllowanceMs: number; }; derivedBudgetsMs: CalibratedColdPathBudget; @@ -492,8 +505,8 @@ describe("full-E2E cold-path calibration", () => { }); }); - // source-shape-contract: compatibility -- Same-head run evidence keeps the local-build allowance bounded and reproducible - it("keeps the authoritative local-build allowance tied to same-head PR evidence", () => { + // source-shape-contract: compatibility -- Exact PR run evidence keeps the local-build allowance bounded and reproducible + it("keeps the authoritative local-build allowance tied to exact PR evidence", () => { const adjustment = calibration.authoritativeLocalBaseBuildAdjustment; expect(adjustment.validatedAt).toMatch(/^\d{4}-\d{2}-\d{2}$/u); expect(adjustment.triggerOutput).toContain("Building OpenClaw sandbox base image locally"); @@ -502,12 +515,49 @@ describe("full-E2E cold-path calibration", () => { "nemoclaw.onboard.phase.sandbox", ]); expect(adjustment.derivation.statistic).toBe("maximum-budget-excess"); - expect(adjustment.runs).toHaveLength(2); - expect(new Set(adjustment.runs.map((run) => run.headSha)).size).toBe(1); + expect(adjustment.runs.length).toBeGreaterThanOrEqual(2); + expect(new Set(adjustment.runs.map((run) => run.runId)).size).toBe(adjustment.runs.length); for (const run of adjustment.runs) { expect(run.runUrl).toBe(`https://github.com/NVIDIA/NemoClaw/actions/runs/${run.runId}`); expect(run.headSha).toMatch(/^[0-9a-f]{40}$/u); + expect(run.triggerEvidence).toEqual({ + artifact: "e2e-full-e2e", + path: "full-e2e-install-onboard-inference-cli-operations-and-cleanup/shell/phase-1-install-sh.stderr.txt", + output: adjustment.triggerOutput, + }); + expect(new Set(run.nativeSecurityInputPaths).size).toBe(run.nativeSecurityInputPaths.length); } + expect( + adjustment.runs.map((run) => ({ + headSha: run.headSha, + nativeSecurityInputPaths: run.nativeSecurityInputPaths, + })), + ).toEqual([ + { + headSha: "188d9a75b3e5efdafeb38e885138bb196197574f", + nativeSecurityInputPaths: [], + }, + { + headSha: "188d9a75b3e5efdafeb38e885138bb196197574f", + nativeSecurityInputPaths: [], + }, + { + headSha: "5f190e4948a11f8b05655e085c11803ce0a9a0a8", + nativeSecurityInputPaths: [ + "Dockerfile.base", + "scripts/security/build-native-security-packages.sh", + "scripts/security/patches/libssh2-1.11.1-cve-2026.patch", + "scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch", + ], + }, + ]); + expect(adjustment.retirement).toEqual({ + trigger: "successful-single-sha-calibration", + minimumSampleCount: 5, + allSamplesSameHead: true, + nativeSecurityInputsMustBeUnchanged: true, + action: "replace-baseline-and-remove-adjustment", + }); const maximumExcessMs = Math.max( ...adjustment.runs.flatMap((run) => [ run.rootStartToFirstTurnCompletionMs - diff --git a/test/perl-critical-cve-remediation.test.ts b/test/perl-critical-cve-remediation.test.ts index 98f6eade210..466c1a4215a 100644 --- a/test/perl-critical-cve-remediation.test.ts +++ b/test/perl-critical-cve-remediation.test.ts @@ -27,6 +27,22 @@ function completedStage(): string { return dockerfile.slice(start); } +function runInstructionContaining(stage: string, needle: string): string { + const lines = stage.split("\n"); + const needleLine = lines.findIndex((line) => line.includes(needle)); + expect(needleLine, `missing ${needle}`).toBeGreaterThanOrEqual(0); + let start = needleLine; + while (start >= 0 && !lines[start]?.startsWith("RUN ")) { + start -= 1; + } + expect(start, `missing RUN instruction for ${needle}`).toBeGreaterThanOrEqual(0); + let end = start; + while (end < lines.length - 1 && lines[end]?.trimEnd().endsWith("\\")) { + end += 1; + } + return lines.slice(start, end + 1).join("\n"); +} + function argumentDefault(name: string): string | undefined { return dockerfile.match(new RegExp(`^ARG ${name}=([^\\s]+)$`, "mu"))?.[1]; } @@ -123,6 +139,10 @@ describe("sandbox base critical Perl CVE remediation", () => { it("replaces the vulnerable distro packages without breaking dpkg ownership (#7338)", () => { const builder = stageNamed("perl-builder"); const runtime = completedStage(); + const perlInstall = runInstructionContaining( + runtime, + "/tmp/nemoclaw-native-security/perl-base.deb", + ); expect(builder).toContain( '"Provides: libperl5.40 (= ${package_version}), perl-modules-5.40 (= ${package_version})"', @@ -131,9 +151,9 @@ describe("sandbox base critical Perl CVE remediation", () => { expect(builder).toContain( '"Replaces: libperl5.40, perl-modules-5.40, perl (<< ${package_version})"', ); - expect(runtime).toContain( - "apt-get install -y --no-install-recommends /tmp/perl-base.deb /tmp/perl.deb", - ); + expect(perlInstall.match(/apt-get install -y --no-install-recommends/g)).toHaveLength(1); + expect(perlInstall).toContain("/tmp/nemoclaw-native-security/perl-base.deb"); + expect(perlInstall).toContain("/tmp/nemoclaw-native-security/perl.deb"); expect(runtime).toContain('test -z "$(dpkg --audit)"'); }); diff --git a/test/sandbox-base-runtime-tools.test.ts b/test/sandbox-base-runtime-tools.test.ts new file mode 100644 index 00000000000..20ed95a3e55 --- /dev/null +++ b/test/sandbox-base-runtime-tools.test.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + BASE_APT_SECURITY_FUNCTIONS, + dockerRunCommandBetween, + runLoggedDockerShell, +} from "./helpers/base-apt-security-functions"; +import { stageFixedParser, useRealPatchedParser } from "./helpers/python-parser-security-fixture"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const DOCKERFILE_BASE = path.join(ROOT, "Dockerfile.base"); +const fixtures: string[] = []; + +function runBaseAptLayer(prefix: string) { + const source = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); + const completedStage = source.lastIndexOf("\nFROM "); + const dockerfile = completedStage >= 0 ? source.slice(completedStage) : source; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + fixtures.push(tmp); + const lists = path.join(tmp, "apt-lists"); + const debianSecurityDebs = path.join(tmp, "debian-security-debs"); + const nativeSecurityDebs = path.join(tmp, "native-security-debs"); + const fakePythonLink = path.join(tmp, "usr-local-bin", "python"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); + fs.mkdirSync(lists); + fs.mkdirSync(debianSecurityDebs); + fs.mkdirSync(nativeSecurityDebs); + fs.mkdirSync(path.dirname(fakePythonLink), { recursive: true }); + fs.writeFileSync(path.join(nativeSecurityDebs, "libssh2-1t64.deb"), "fixed libssh2"); + fs.writeFileSync( + path.join(nativeSecurityDebs, "nemoclaw-python3.13-htmlparser-fix.deb"), + "fixed parser package", + ); + const command = dockerRunCommandBetween( + dockerfile, + "RUN apt-get update", + "# gosu for privilege separation", + ) + .replaceAll("/var/lib/apt/lists", lists) + .replaceAll("/tmp/nemoclaw-debian-security", debianSecurityDebs) + .replaceAll("/tmp/nemoclaw-native-security", nativeSecurityDebs) + .replaceAll("/usr/local/share/nemoclaw", path.join(tmp, "security-inventory")) + .replaceAll("/usr/local/bin/python", fakePythonLink) + .replaceAll("/usr/bin/python3", pythonShim) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); + const result = runLoggedDockerShell(command, tmp, [ + 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', + 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', + 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', + ...useRealPatchedParser(BASE_APT_SECURITY_FUNCTIONS, pythonShim), + ]); + const calls = fs.readFileSync(path.join(tmp, "calls.log"), "utf-8"); + return { calls, fakePythonLink, pythonShim, result }; +} + +afterEach(() => { + for (const fixture of fixtures.splice(0)) { + fs.rmSync(fixture, { recursive: true, force: true }); + } +}); + +describe("sandbox base runtime tools", () => { + it("installs the required process, filesystem, and SFTP tools", () => { + const { calls, result } = runBaseAptLayer("nemoclaw-base-apt-"); + + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); + expect(calls).toContain("procps=2:4.0.4-9"); + expect(calls).toContain("e2fsprogs=1.47.2-3+b11"); + expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u4"); + }); + + it("symlinks bare `python` to the tested python3 interpreter (#1452)", () => { + const { fakePythonLink, pythonShim, result } = runBaseAptLayer("nemoclaw-base-pysymlink-"); + + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); + expect(fs.lstatSync(fakePythonLink).isSymbolicLink()).toBe(true); + expect(fs.readlinkSync(fakePythonLink)).toBe(pythonShim); + }); +}); diff --git a/test/sandbox-base-security-packages.test.ts b/test/sandbox-base-security-packages.test.ts index e4ac64d8991..3c286da244b 100644 --- a/test/sandbox-base-security-packages.test.ts +++ b/test/sandbox-base-security-packages.test.ts @@ -11,6 +11,7 @@ import { dockerRunCommandBetween, runLoggedDockerShell, } from "./helpers/base-apt-security-functions"; +import { stageFixedParser, useRealPatchedParser } from "./helpers/python-parser-security-fixture"; const ROOT = path.resolve(import.meta.dirname, ".."); const SECURITY_IMAGES = [ @@ -44,27 +45,41 @@ const SECURITY_CASES = SECURITY_IMAGES.flatMap((image) => function sandboxSecurityCommand( image: (typeof SECURITY_IMAGES)[number], tmp: string, -): { command: string; inventory: string; securityDebs: string } { +): { + command: string; + inventory: string; + debianSecurityDebs: string; + nativeSecurityDebs: string; + pythonShim: string; +} { const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); + const debianSecurityDebs = path.join(tmp, "debian-security-debs"); + const nativeSecurityDebs = path.join(tmp, "native-security-debs"); const inventoryDirectory = path.join(tmp, "security-inventory"); const inventory = path.join(inventoryDirectory, "security-packages.txt"); - const fakePython3 = path.join(tmp, "usr-bin", "python3"); const fakePythonLink = path.join(tmp, "usr-local-bin", "python"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); fs.mkdirSync(lists); - fs.mkdirSync(path.dirname(fakePython3), { recursive: true }); + fs.mkdirSync(debianSecurityDebs); + fs.mkdirSync(nativeSecurityDebs); fs.mkdirSync(path.dirname(fakePythonLink), { recursive: true }); - fs.writeFileSync(fakePython3, "#!/bin/sh\n", { mode: 0o755 }); + fs.writeFileSync(path.join(nativeSecurityDebs, "libssh2-1t64.deb"), "fixed libssh2"); + fs.writeFileSync( + path.join(nativeSecurityDebs, "nemoclaw-python3.13-htmlparser-fix.deb"), + "fixed parser package", + ); const dockerfile = fs.readFileSync(image.dockerfile, "utf-8"); const command = dockerRunCommandBetween(dockerfile, image.startMarker, image.endMarker) .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) + .replaceAll("/tmp/nemoclaw-debian-security", debianSecurityDebs) + .replaceAll("/tmp/nemoclaw-native-security", nativeSecurityDebs) .replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory) .replaceAll("/usr/local/share/nemoclaw", inventoryDirectory) .replaceAll("/usr/local/bin/python", fakePythonLink) - .replaceAll("/usr/bin/python3", fakePython3); - return { command, inventory, securityDebs }; + .replaceAll("/usr/bin/python3", pythonShim) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); + return { command, inventory, debianSecurityDebs, nativeSecurityDebs, pythonShim }; } function securityInventory(architecture: (typeof ARCHITECTURES)[number]): string { @@ -76,6 +91,8 @@ function securityInventory(architecture: (typeof ARCHITECTURES)[number]): string "jq=1.8.2-1", "vim-common=2:9.2.0782-1", "vim-tiny=2:9.2.0782-1", + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1", + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1", "", ].join("\n"); } @@ -84,16 +101,19 @@ function completedImageSecurityCommand( image: (typeof SECURITY_IMAGES)[number], tmp: string, architecture: (typeof ARCHITECTURES)[number], -): { command: string; inventory: string } { +): { command: string; inventory: string; pythonShim: string } { const inventory = path.join(tmp, "security-packages.txt"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); fs.writeFileSync(inventory, securityInventory(architecture), { mode: 0o444 }); const dockerfile = fs.readFileSync(image.finalDockerfile, "utf-8"); const command = dockerRunCommandBetween( dockerfile, "# Verify the immutable security package inventory in the completed image.", "# End completed-image security package verification.", - ).replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory); - return { command, inventory }; + ) + .replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); + return { command, inventory, pythonShim }; } describe("sandbox base security packages", () => { @@ -101,20 +121,20 @@ describe("sandbox base security packages", () => { SECURITY_CASES, )("executes the exact security package contract for %s on %s", (_name, architecture, image) => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-security-")); - const { command, inventory, securityDebs } = sandboxSecurityCommand(image, tmp); + const prepared = sandboxSecurityCommand(image, tmp); try { - const result = runLoggedDockerShell(command, tmp, [ + const result = runLoggedDockerShell(prepared.command, tmp, [ 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...baseAptSecurityFunctions(architecture), + ...useRealPatchedParser(baseAptSecurityFunctions(architecture), prepared.pythonShim), ]); expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); const calls = fs.readFileSync(path.join(tmp, "calls.log"), "utf-8"); expect(calls).toContain("dpkg-install"); - expect(fs.readFileSync(inventory, "utf-8")).toBe(securityInventory(architecture)); - expect(fs.statSync(inventory).mode & 0o777).toBe(0o444); + expect(fs.readFileSync(prepared.inventory, "utf-8")).toBe(securityInventory(architecture)); + expect(fs.statSync(prepared.inventory).mode & 0o777).toBe(0o444); expect( calls .split("\n") @@ -128,7 +148,9 @@ describe("sandbox base security packages", () => { "vim-common_9.2.0782-1_all.deb", `vim-tiny_9.2.0782-1_${architecture}.deb`, ]); - expect(fs.existsSync(securityDebs)).toBe(false); + expect(prepared.debianSecurityDebs).not.toBe(prepared.nativeSecurityDebs); + expect(fs.existsSync(prepared.debianSecurityDebs)).toBe(false); + expect(fs.existsSync(prepared.nativeSecurityDebs)).toBe(image.name === "OpenClaw"); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } @@ -148,7 +170,7 @@ describe("sandbox base security packages", () => { ' printf "0:0:444\\n"', "}", ].join("\n"), - ...baseAptSecurityFunctions(architecture), + ...useRealPatchedParser(baseAptSecurityFunctions(architecture), prepared.pythonShim), ]); expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); } finally { @@ -169,11 +191,12 @@ describe("sandbox base security packages", () => { try { const result = runLoggedDockerShell(command, tmp, [ 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', - ...baseAptSecurityFunctions(architecture), + ...useRealPatchedParser(baseAptSecurityFunctions(architecture), prepared.pythonShim), ]); expect(result.status).not.toBe(0); expect(fs.readFileSync(path.join(tmp, "calls.log"), "utf-8")).not.toContain("dpkg-install"); - expect(fs.existsSync(prepared.securityDebs)).toBe(true); + expect(fs.existsSync(prepared.debianSecurityDebs)).toBe(true); + expect(fs.existsSync(prepared.nativeSecurityDebs)).toBe(true); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index a6d4b5e0c93..b5c15e1a9dc 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -14,7 +14,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { BASE_APT_SECURITY_FUNCTIONS } from "./helpers/base-apt-security-functions"; const ROOT = path.resolve(import.meta.dirname, ".."); const DOCKERFILE = path.join(ROOT, "Dockerfile"); @@ -1051,80 +1050,6 @@ describe("sandbox provisioning: base runtime tools", () => { expect(aptInstall).toBeDefined(); expect(aptInstall).toContain("nftables=1.1.3-1"); }); - it("base apt layer requests procps, e2fsprogs, and the SFTP server", () => { - const dockerfile = completedDockerStage(fs.readFileSync(DOCKERFILE_BASE, "utf-8")); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-apt-")); - const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); - const fakePy3 = path.join(tmp, "usr-bin", "python3"); - const fakePyLink = path.join(tmp, "usr-local-bin", "python"); - fs.mkdirSync(lists); - fs.mkdirSync(path.dirname(fakePy3), { recursive: true }); - fs.mkdirSync(path.dirname(fakePyLink), { recursive: true }); - fs.writeFileSync(fakePy3, "#!/bin/sh\n", { mode: 0o755 }); - const command = dockerRunCommandBetween( - dockerfile, - "RUN apt-get update", - "# gosu for privilege separation", - ) - .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) - .replaceAll("/usr/local/share/nemoclaw", path.join(tmp, "security-inventory")) - .replaceAll("/usr/local/bin/python", fakePyLink) - .replaceAll("/usr/bin/python3", fakePy3); - try { - const { result, calls } = runLoggedDockerShell(command, tmp, [ - 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', - 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', - 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...BASE_APT_SECURITY_FUNCTIONS, - ]); - expect(result.status).toBe(0); - expect(calls).toContain("apt-get update"); - expect(calls).toContain("procps=2:4.0.4-9"); - expect(calls).toContain("e2fsprogs=1.47.2-3+b11"); - expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u4"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("symlinks bare `python` to python3 so agent tool calls don't fail with command-not-found (#1452)", () => { - const dockerfile = completedDockerStage(fs.readFileSync(DOCKERFILE_BASE, "utf-8")); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-pysymlink-")); - const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); - const fakePy3 = path.join(tmp, "usr-bin", "python3"); - const fakePyLink = path.join(tmp, "usr-local-bin", "python"); - fs.mkdirSync(lists, { recursive: true }); - fs.mkdirSync(path.dirname(fakePy3), { recursive: true }); - fs.mkdirSync(path.dirname(fakePyLink), { recursive: true }); - fs.writeFileSync(fakePy3, "#!/bin/sh\necho 3.13\n", { mode: 0o755 }); - const command = dockerRunCommandBetween( - dockerfile, - "RUN apt-get update", - "# gosu for privilege separation", - ) - .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) - .replaceAll("/usr/local/share/nemoclaw", path.join(tmp, "security-inventory")) - .replaceAll("/usr/local/bin/python", fakePyLink) - .replaceAll("/usr/bin/python3", fakePy3); - try { - const { result } = runLoggedDockerShell(command, tmp, [ - 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', - 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', - 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...BASE_APT_SECURITY_FUNCTIONS, - ]); - expect(result.status).toBe(0); - expect(fs.lstatSync(fakePyLink).isSymbolicLink()).toBe(true); - expect(fs.readlinkSync(fakePyLink)).toBe(fakePy3); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - it("runtime hardening installs procps and e2fsprogs when a stale base lacks ps and chattr", () => { const dockerfile = fs.readFileSync(DOCKERFILE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-procps-"));