diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 9538cf26f64..0431ae9901a 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -37,6 +37,9 @@ on: - "nemoclaw-blueprint/blueprint.yaml" - "scripts/lib/openclaw-npm-remediation.mts" - "scripts/lib/reviewed-npm-audit.mts" + - "scripts/security/build-native-security-packages.sh" + - "scripts/security/patches/libssh2-1.11.1-cve-2026.patch" + - "scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch" - "scripts/checks/node-tar-image-scan.mts" - "scripts/patch-bundled-npm-brace-expansion.mts" - "scripts/patch-bundled-npm-tar.mts" diff --git a/Dockerfile b/Dockerfile index 9ad859406ad..5d6f3c6338b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1736,6 +1736,8 @@ RUN set -eu; \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ | cmp -s - "$security_inventory"; \ test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1"; \ @@ -1743,10 +1745,18 @@ RUN set -eu; \ test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1"; \ test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1"; \ + test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1"; \ + test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1"; \ ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5'; \ test "$(jq --version)" = "jq-1.8.2"; \ printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null; \ python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; \ + printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c -; \ + python3 -c "from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''"; \ + python3 -c "import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'"; \ vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 '; \ test -z "$(dpkg --audit)" # End completed-image security package verification. diff --git a/Dockerfile.base b/Dockerfile.base index d5630581681..15a5145f835 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -56,6 +56,28 @@ ARG PERL_VERSION=5.44.0 ARG PERL_SHA256=505cf43912e9480495c344c70260452e32aa2a73c546a026b3f100053b23ce91 ARG PERL_PACKAGE_REVISION=1nemoclaw1 +# Debian trixie has not published libssh2 or Python packages containing the +# reviewed upstream fixes. Build native packages once and install the same +# artifacts into the managed OpenClaw image. +FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba AS native-security-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.12 \ + ca-certificates=20250419 \ + curl=8.14.1-2+deb13u4 \ + git=1:2.47.3-0+deb13u1 \ + libssl-dev=3.5.6-1~deb13u2 \ + openssh-server=1:10.0p1-7+deb13u4 \ + xz-utils=5.8.1-1+deb13u1 \ + zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh +COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch +COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch + +RUN bash /scripts/security/build-native-security-packages.sh /out + # Debian trixie has not published a Perl package containing the upstream # fixes for CVE-2026-12087, CVE-2026-13221, and CVE-2026-57433. Build the # fixed upstream release as native Debian packages so dpkg dependencies and @@ -66,6 +88,8 @@ ARG PERL_VERSION ARG PERL_SHA256 ARG PERL_PACKAGE_REVISION +COPY --from=native-security-builder /out /tmp/security-packages + RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential=12.12 \ ca-certificates=20250419 \ @@ -157,11 +181,15 @@ RUN package_version="${PERL_VERSION}-${PERL_PACKAGE_REVISION}" \ 'Maintainer: NVIDIA NemoClaw Maintainers' \ 'Description: Perl 5 language interpreter metapackage for the NemoClaw sandbox' \ > /tmp/perl-meta/DEBIAN/control \ - && dpkg-deb --build --root-owner-group /tmp/perl-root /tmp/perl-base.deb \ - && dpkg-deb --build --root-owner-group /tmp/perl-meta /tmp/perl.deb + && dpkg-deb --build --root-owner-group \ + /tmp/perl-root /tmp/security-packages/perl-base.deb \ + && dpkg-deb --build --root-owner-group \ + /tmp/perl-meta /tmp/security-packages/perl.deb FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba +COPY --from=perl-builder /tmp/security-packages /tmp/nemoclaw-native-security + # OpenShell blocks the link-local EC2 Instance Metadata Service. Keep AWS SDK # credential chains from attempting an impossible metadata discovery path. ENV AWS_EC2_METADATA_DISABLED=true @@ -254,17 +282,27 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "$security_deb_dir/jq.deb" \ "$security_deb_dir/vim-common.deb" \ "$security_deb_dir/vim-tiny.deb" \ + /tmp/nemoclaw-native-security/libssh2-1t64.deb \ + /tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb \ && test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1" \ && test "$(dpkg-query -W -f='${Version}' libjq1)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1" \ && test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1" \ + && test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1" \ + && test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1" \ && test "$(dpkg-query -W -f='${Version}' perl)" = "$perl_version_before" \ && test "$(jq --version)" = "jq-1.8.2" \ && ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5' \ && printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null \ && python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION" \ + && printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c - \ + && python3 -c "from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''" \ + && python3 -c "import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'" \ && vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 ' \ && install -d -o root -g root -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' \ @@ -275,6 +313,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ > /usr/local/share/nemoclaw/security-packages.txt \ && chown root:root /usr/local/share/nemoclaw/security-packages.txt \ && chmod 0444 /usr/local/share/nemoclaw/security-packages.txt \ @@ -282,11 +322,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && ln -s /usr/bin/python3 /usr/local/bin/python -COPY --from=perl-builder /tmp/perl-base.deb /tmp/perl.deb /tmp/ - RUN apt-get update \ - && apt-get install -y --no-install-recommends /tmp/perl-base.deb /tmp/perl.deb \ - && rm -f /tmp/perl-base.deb /tmp/perl.deb \ + && apt-get install -y --no-install-recommends \ + /tmp/nemoclaw-native-security/perl-base.deb \ + /tmp/nemoclaw-native-security/perl.deb \ + && rm -rf /tmp/nemoclaw-native-security \ && rm -rf /var/lib/apt/lists/* \ && test "$(perl -e 'print $^V')" = "v5.44.0" \ && test "$(perl -MSocket -e 'print Socket->VERSION')" = "2.041" \ diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 175e54201be..434e7d4c77f 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -762,6 +762,8 @@ RUN set -eu; \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ | cmp -s - "$security_inventory"; \ test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1"; \ @@ -769,10 +771,18 @@ RUN set -eu; \ test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1"; \ test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1"; \ + test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1"; \ + test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1"; \ ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5'; \ test "$(jq --version)" = "jq-1.8.2"; \ printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null; \ python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; \ + printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c -; \ + python3 -c "from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''"; \ + python3 -c "import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'"; \ vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 '; \ test -z "$(dpkg --audit)" # End completed-image security package verification. diff --git a/agents/hermes/Dockerfile.base b/agents/hermes/Dockerfile.base index 40d5a083115..ef6d52a5f53 100644 --- a/agents/hermes/Dockerfile.base +++ b/agents/hermes/Dockerfile.base @@ -19,8 +19,29 @@ # 6. New .hermes subdirectory — add mkdir/chmod below # ──────────────────────────────────────────────────────────────── +FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba AS native-security-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.12 \ + ca-certificates=20250419 \ + curl=8.14.1-2+deb13u4 \ + git=1:2.47.3-0+deb13u1 \ + libssl-dev=3.5.6-1~deb13u2 \ + openssh-server=1:10.0p1-7+deb13u4 \ + xz-utils=5.8.1-1+deb13u1 \ + zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh +COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch +COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch + +RUN bash /scripts/security/build-native-security-packages.sh /out + FROM node:24-trixie-slim@sha256:05c08ce4291e9a58f59456a7985176defb12cdd42271f35ff81a3e167ea61d4c +COPY --from=native-security-builder /out /tmp/nemoclaw-native-security + ENV DEBIAN_FRONTEND=noninteractive # Hermes version pinned for reproducibility. All four values below are @@ -122,16 +143,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "$security_deb_dir/jq.deb" \ "$security_deb_dir/vim-common.deb" \ "$security_deb_dir/vim-tiny.deb" \ + /tmp/nemoclaw-native-security/libssh2-1t64.deb \ + /tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb \ && test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1" \ && test "$(dpkg-query -W -f='${Version}' libjq1)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1" \ && test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1" \ + && test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1" \ + && test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1" \ && test "$(jq --version)" = "jq-1.8.2" \ && ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5' \ && printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null \ && python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION" \ + && printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c - \ + && python3 -c "from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''" \ + && python3 -c "import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'" \ && vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 ' \ && install -d -o root -g root -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' \ @@ -142,9 +173,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ > /usr/local/share/nemoclaw/security-packages.txt \ && chown root:root /usr/local/share/nemoclaw/security-packages.txt \ && chmod 0444 /usr/local/share/nemoclaw/security-packages.txt \ + && rm -rf /tmp/nemoclaw-native-security \ && rm -rf "$security_deb_dir" \ && rm -rf /var/lib/apt/lists/* diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index 84453d1cad0..18da7fa9e67 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -237,6 +237,8 @@ RUN set -eu; \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ | cmp -s - "$security_inventory"; \ test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1"; \ @@ -244,10 +246,18 @@ RUN set -eu; \ test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1"; \ test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1"; \ test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1"; \ + test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1"; \ + test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1"; \ ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5'; \ test "$(jq --version)" = "jq-1.8.2"; \ printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null; \ python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; \ + printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c -; \ + python3 -c "from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''"; \ + python3 -c "import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'"; \ vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 '; \ test -z "$(dpkg --audit)" # End completed-image security package verification. diff --git a/agents/langchain-deepagents-code/Dockerfile.base b/agents/langchain-deepagents-code/Dockerfile.base index 63e5f9449c4..e9cef60d0c6 100644 --- a/agents/langchain-deepagents-code/Dockerfile.base +++ b/agents/langchain-deepagents-code/Dockerfile.base @@ -7,8 +7,29 @@ # Node for NemoClaw build-time config generation, Python, shell tools, and a # hash-locked deepagents-code install with the NVIDIA provider extra. +FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba AS native-security-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential=12.12 \ + ca-certificates=20250419 \ + curl=8.14.1-2+deb13u4 \ + git=1:2.47.3-0+deb13u1 \ + libssl-dev=3.5.6-1~deb13u2 \ + openssh-server=1:10.0p1-7+deb13u4 \ + xz-utils=5.8.1-1+deb13u1 \ + zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \ + && rm -rf /var/lib/apt/lists/* + +COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh +COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch +COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch + +RUN bash /scripts/security/build-native-security-packages.sh /out + FROM node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba +COPY --from=native-security-builder /out /tmp/nemoclaw-native-security + COPY scripts/lib/reviewed-npm-archive.mts /scripts/lib/reviewed-npm-archive.mts COPY scripts/patch-bundled-npm-brace-expansion.mts /scripts/patch-bundled-npm-brace-expansion.mts COPY scripts/patch-bundled-npm-tar.mts /scripts/patch-bundled-npm-tar.mts @@ -97,16 +118,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "$security_deb_dir/jq.deb" \ "$security_deb_dir/vim-common.deb" \ "$security_deb_dir/vim-tiny.deb" \ + /tmp/nemoclaw-native-security/libssh2-1t64.deb \ + /tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb \ && test "$(dpkg-query -W -f='${Version}' libexpat1)" = "2.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' libonig5)" = "6.9.9-1+b1" \ && test "$(dpkg-query -W -f='${Version}' libjq1)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' jq)" = "1.8.2-1" \ && test "$(dpkg-query -W -f='${Version}' vim-common)" = "2:9.2.0782-1" \ && test "$(dpkg-query -W -f='${Version}' vim-tiny)" = "2:9.2.0782-1" \ + && test "$(dpkg-query -W -f='${Version}' libssh2-1t64)" = "1.11.1-1+deb13u1+nemoclaw1" \ + && test "$(dpkg-query -W -f='${Version}' nemoclaw-python3.13-htmlparser-fix)" = "3.13.5-2+deb13u4+nemoclaw1" \ && test "$(jq --version)" = "jq-1.8.2" \ && ldd /usr/bin/jq | grep -Eq 'libonig[.]so[.]5' \ && printf '%s\n' '{"sandbox":"healthy"}' | jq -e '.sandbox == "healthy"' >/dev/null \ && python3 -c "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION" \ + && printf '%s %s\n' \ + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" \ + /usr/lib/python3.13/html/parser.py \ + | sha256sum -c - \ + && python3 -c "from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''" \ + && python3 -c "import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'" \ && vim.tiny --version | head -n 1 | grep -Eq '^VIM - Vi IMproved 9[.]2 ' \ && install -d -o root -g root -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' \ @@ -117,9 +148,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ "jq=1.8.2-1" \ "vim-common=2:9.2.0782-1" \ "vim-tiny=2:9.2.0782-1" \ + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" \ + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" \ > /usr/local/share/nemoclaw/security-packages.txt \ && chown root:root /usr/local/share/nemoclaw/security-packages.txt \ && chmod 0444 /usr/local/share/nemoclaw/security-packages.txt \ + && rm -rf /tmp/nemoclaw-native-security \ && rm -rf "$security_deb_dir" \ && rm -rf /var/lib/apt/lists/* diff --git a/docs/security/sandbox-base-2026-07-27-native-package-review.md b/docs/security/sandbox-base-2026-07-27-native-package-review.md new file mode 100644 index 00000000000..bfd2d2234fc --- /dev/null +++ b/docs/security/sandbox-base-2026-07-27-native-package-review.md @@ -0,0 +1,95 @@ + + + +# Sandbox base native package review: libssh2 and Python HTMLParser + +Date: 2026-07-27 + +## Scope + +This review covers two temporary native-package remediations shared by the OpenClaw, Hermes, and Deep Agents Code base images: + +- backport the reviewed upstream fixes for CVE-2026-66032, CVE-2026-66033, CVE-2026-66034, and CVE-2026-66035 to libssh2 1.11.1; and +- backport the reviewed CPython 3.13 fix for CVE-2026-15308 to Debian's Python 3.13.5 standard library. + +The supported Debian suite has not published packages containing these fixes. +The changes preserve the existing image behavior and do not create a new integration or product surface. + +## Reviewed identities + +| Component | Input identity | Fixed identity | Upstream fix | +| --- | --- | --- | --- | +| libssh2 | `libssh2-1t64=1.11.1-1+deb13u1` | `libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1` | `5e4776146552d898b9c0e1b313cd093fa8dc92d0`, `a2ed82d40964bbc0d64cd717aa0a5a892117d2e6`, `a13bb6c773f0d55ad1628cede57e99803cd898d9`, and `42e33d81577ed4b95d4b4f6f845e5ee8efe5eeb4` | +| Python HTMLParser | `libpython3.13-stdlib=3.13.5-2+deb13u4` | `nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1` | `7933f4bf7131aa4140750f9404f5de0aa2969ced` | + +The libssh2 source archive is bound to SHA-256 `9954cb54c4f548198a7cbebad248bdc87dd64bd26185708a294b2b50771e3769`. +The original Debian package SHA-256 values are: + +| Package | amd64 | arm64 | +| --- | --- | --- | +| `libssh2-1t64_1.11.1-1+deb13u1` | `915c4ec450a369d430e0151f9e10e25044ea2f0d6e41901e00a9317e232e5683` | `600c2a845d6d14d292c765382bc7e644898762e1634a4aecf5b85329622dbbfe` | +| `libpython3.13-stdlib_3.13.5-2+deb13u4` | `0def2d972310b59704ad119abee5a97f95409e14ff1359edd8cc7b8892cfd43f` | `37cce6086b7c1ca93086f83b68761737607689e634693b6972b5dbfd6c080872` | + +The unmodified Debian `html/parser.py` input is bound to SHA-256 `f91ec3de6331206bbe2ec3e54a05f646bd23d3c61a18d4a01b25164e070bacc9`. +The fixed file is bound to SHA-256 `4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7`. +The file contains the reviewed upstream batching fix plus an empty-input guard so repeated `feed("")` calls remain no-ops instead of accumulating pending entries. + +## Package contracts + +The shared builder applies the reviewed patches only after verifying the source artifacts. +It builds libssh2 with OpenSSL and zlib support, runs the upstream `make check` suite, verifies the `libssh2.so.1` soname, and rejects a build that removes any exported symbol from the original Debian library. +It then replaces only the shared library in the original Debian runtime package and records the NemoClaw package revision. + +The Python fix package owns only `/usr/lib/python3.13/html/parser.py`. +It depends on the exact Debian `libpython3.13-stdlib=3.13.5-2+deb13u4` input and declares the narrow file replacement for that version. +This avoids changing Python's exact internal package dependency chain while making the fixed file visible as a separate dpkg identity. + +Every managed base image: + +1. installs both native packages; +2. verifies their exact dpkg versions; +3. verifies the fixed HTMLParser file hash; +4. verifies that `html.parser` resolves to the exact hashed file; +5. verifies 20,000 empty `feed()` calls leave no pending entries and exercises incremental HTML comment parsing across 20,000 input chunks; +6. loads `libssh2.so.1` and verifies the 1.11.1 runtime identity; +7. records both packages in the root-owned, read-only security inventory; and +8. removes the temporary package artifacts. + +The completed production images repeat the inventory, package, file, and runtime checks and require an empty `dpkg --audit` result. + +## Concern ledger + +### SEC-1: libssh2 1.11.1 lacks four upstream memory-safety fixes + +- Surface: native SSH2 client library +- Severity: high +- Confidence: high +- Failure mode: malformed SSH, SFTP, or public-key data can reach missing bounds or lifetime checks. +- Disposition: backport, build, test, package, runtime-proof +- Remaining gate: native amd64 and arm64 image builds for all managed images. + +### SEC-2: Python 3.13.5 HTMLParser repeatedly rescans incomplete input + +- Surface: Python standard library +- Severity: high +- Confidence: high +- Failure mode: repeated small `feed()` calls with an unterminated construct can cause quadratic CPU consumption. +- Disposition: backport, package, file-hash-proof, behavior-test +- Remaining gate: native amd64 and arm64 image builds for all managed images. + +## Removal conditions + +Remove the libssh2 backport after the supported Debian suite publishes a package containing all four reviewed commits and that package passes the same symbol, upstream test, runtime, and multi-architecture image checks. + +Remove the Python file replacement after the supported Debian suite publishes a Python 3.13 package containing commit `7933f4bf7131aa4140750f9404f5de0aa2969ced` or its reviewed successor and the replacement passes the same incremental-parser and multi-architecture image checks. + +## Verification + +Required evidence for the final pull-request head: + +- shared-builder syntax and immutable-input contract tests; +- exact base-package and completed-image contract tests for all three managed images on amd64 and arm64; +- the full libssh2 upstream test suite in each native builder; +- symbol and soname compatibility against the original Debian libssh2 package; +- repository formatting and type checks; and +- successful native base-image builds for all managed images. diff --git a/scripts/security/build-native-security-packages.sh b/scripts/security/build-native-security-packages.sh new file mode 100755 index 00000000000..f3dd7643e33 --- /dev/null +++ b/scripts/security/build-native-security-packages.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly LIBSSH2_VERSION="1.11.1" +readonly LIBSSH2_DEBIAN_VERSION="1.11.1-1+deb13u1" +readonly LIBSSH2_PACKAGE_VERSION="${LIBSSH2_DEBIAN_VERSION}+nemoclaw1" +readonly LIBSSH2_SOURCE_SHA256="9954cb54c4f548198a7cbebad248bdc87dd64bd26185708a294b2b50771e3769" +readonly PYTHON_DEBIAN_VERSION="3.13.5-2+deb13u4" +readonly PYTHON_FIX_VERSION="${PYTHON_DEBIAN_VERSION}+nemoclaw1" +readonly PYTHON_PARSER_SHA256="f91ec3de6331206bbe2ec3e54a05f646bd23d3c61a18d4a01b25164e070bacc9" +readonly PYTHON_PARSER_FIXED_SHA256="4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly script_dir +readonly patch_dir="${script_dir}/patches" +readonly output_dir="${1:-/out}" +build_root="$(mktemp -d /tmp/nemoclaw-native-security.XXXXXX)" +readonly build_root + +cleanup() { + rm -rf "${build_root}" +} +trap cleanup EXIT + +download() { + local url="$1" + local output="$2" + curl --proto '=https' --tlsv1.2 -fsSL \ + --retry 5 --retry-all-errors --retry-delay 2 \ + --connect-timeout 15 --max-time 120 \ + -o "${output}" "${url}" +} + +verify_sha256() { + local expected="$1" + local path="$2" + printf '%s %s\n' "${expected}" "${path}" | sha256sum -c - +} + +rewrite_control_field() { + local control="$1" + local field="$2" + local value="$3" + sed -i "s/^${field}: .*$/${field}: ${value}/" "${control}" + grep -Fqx "${field}: ${value}" "${control}" +} + +refresh_md5sums() { + local package_root="$1" + ( + cd "${package_root}" + find . -path ./DEBIAN -prune -o -type f -printf '%P\0' \ + | sort -z \ + | xargs -0 md5sum + ) >"${package_root}/DEBIAN/md5sums" +} + +build_libssh2_package() { + local architecture="$1" + local original_sha256 + local original_deb="${build_root}/libssh2-original.deb" + local source_archive="${build_root}/libssh2.tar.xz" + local source_dir="${build_root}/libssh2-source" + local install_root="${build_root}/libssh2-install" + local package_root="${build_root}/libssh2-package" + local multiarch + + case "${architecture}" in + amd64) + original_sha256="915c4ec450a369d430e0151f9e10e25044ea2f0d6e41901e00a9317e232e5683" + ;; + arm64) + original_sha256="600c2a845d6d14d292c765382bc7e644898762e1634a4aecf5b85329622dbbfe" + ;; + *) + printf 'Unsupported architecture: %s\n' "${architecture}" >&2 + return 64 + ;; + esac + + download \ + "https://libssh2.org/download/libssh2-${LIBSSH2_VERSION}.tar.xz" \ + "${source_archive}" + verify_sha256 "${LIBSSH2_SOURCE_SHA256}" "${source_archive}" + mkdir -p "${source_dir}" + tar -xJf "${source_archive}" -C "${source_dir}" --strip-components=1 + git -C "${source_dir}" apply --check \ + "${patch_dir}/libssh2-1.11.1-cve-2026.patch" + git -C "${source_dir}" apply \ + "${patch_dir}/libssh2-1.11.1-cve-2026.patch" + + ( + cd "${source_dir}" + ./configure \ + --prefix=/usr \ + --disable-static \ + --with-crypto=openssl \ + --with-libz + make -j"$(nproc)" + make check + make DESTDIR="${install_root}" install + ) + + multiarch="$(gcc -print-multiarch)" + test -n "${multiarch}" + test -f "${install_root}/usr/lib/libssh2.so.1.0.1" + + download \ + "https://deb.debian.org/debian-security/pool/updates/main/libs/libssh2/libssh2-1t64_${LIBSSH2_DEBIAN_VERSION}_${architecture}.deb" \ + "${original_deb}" + verify_sha256 "${original_sha256}" "${original_deb}" + dpkg-deb -R "${original_deb}" "${package_root}" + test "$(dpkg-deb -f "${original_deb}" Package)" = "libssh2-1t64" + test "$(dpkg-deb -f "${original_deb}" Version)" = "${LIBSSH2_DEBIAN_VERSION}" + + local original_library="${package_root}/usr/lib/${multiarch}/libssh2.so.1.0.1" + local fixed_library="${install_root}/usr/lib/libssh2.so.1.0.1" + test -f "${original_library}" + readelf -d "${fixed_library}" | grep -Fq '(SONAME)' \ + && readelf -d "${fixed_library}" | grep -Fq '[libssh2.so.1]' + nm -D --defined-only --format=posix "${original_library}" \ + | cut -d ' ' -f 1 | sort -u >"${build_root}/libssh2-original.symbols" + nm -D --defined-only --format=posix "${fixed_library}" \ + | cut -d ' ' -f 1 | sort -u >"${build_root}/libssh2-fixed.symbols" + comm -23 \ + "${build_root}/libssh2-original.symbols" \ + "${build_root}/libssh2-fixed.symbols" \ + >"${build_root}/libssh2-missing.symbols" + test ! -s "${build_root}/libssh2-missing.symbols" + + install -m 0644 "${fixed_library}" "${original_library}" + rewrite_control_field \ + "${package_root}/DEBIAN/control" Version "${LIBSSH2_PACKAGE_VERSION}" + rewrite_control_field \ + "${package_root}/DEBIAN/control" Provides \ + "libssh2-1 (= ${LIBSSH2_PACKAGE_VERSION})" + rewrite_control_field \ + "${package_root}/DEBIAN/control" Breaks \ + "libssh2-1 (<< ${LIBSSH2_PACKAGE_VERSION})" + refresh_md5sums "${package_root}" + dpkg-deb --build --root-owner-group \ + "${package_root}" \ + "${output_dir}/libssh2-1t64.deb" + test "$(dpkg-deb -f "${output_dir}/libssh2-1t64.deb" Version)" = \ + "${LIBSSH2_PACKAGE_VERSION}" +} + +build_python_fix_package() { + local architecture="$1" + local original_sha256 + local original_deb="${build_root}/python-stdlib-original.deb" + local original_root="${build_root}/python-stdlib-original" + local package_root="${build_root}/python-htmlparser-fix" + local parser_path="usr/lib/python3.13/html/parser.py" + + case "${architecture}" in + amd64) + original_sha256="0def2d972310b59704ad119abee5a97f95409e14ff1359edd8cc7b8892cfd43f" + ;; + arm64) + original_sha256="37cce6086b7c1ca93086f83b68761737607689e634693b6972b5dbfd6c080872" + ;; + *) + printf 'Unsupported architecture: %s\n' "${architecture}" >&2 + return 64 + ;; + esac + + download \ + "https://deb.debian.org/debian/pool/main/p/python3.13/libpython3.13-stdlib_${PYTHON_DEBIAN_VERSION}_${architecture}.deb" \ + "${original_deb}" + verify_sha256 "${original_sha256}" "${original_deb}" + dpkg-deb -x "${original_deb}" "${original_root}" + test "$(dpkg-deb -f "${original_deb}" Package)" = "libpython3.13-stdlib" + test "$(dpkg-deb -f "${original_deb}" Version)" = "${PYTHON_DEBIAN_VERSION}" + verify_sha256 "${PYTHON_PARSER_SHA256}" "${original_root}/${parser_path}" + + mkdir -p "${package_root}/DEBIAN" \ + "$(dirname -- "${package_root}/${parser_path}")" + install -m 0644 \ + "${original_root}/${parser_path}" \ + "${package_root}/${parser_path}" + git -C "${package_root}" apply --check \ + "${patch_dir}/python3.13-htmlparser-cve-2026-15308.patch" + git -C "${package_root}" apply \ + "${patch_dir}/python3.13-htmlparser-cve-2026-15308.patch" + verify_sha256 \ + "${PYTHON_PARSER_FIXED_SHA256}" \ + "${package_root}/${parser_path}" + + printf '%s\n' \ + 'Package: nemoclaw-python3.13-htmlparser-fix' \ + "Version: ${PYTHON_FIX_VERSION}" \ + 'Architecture: all' \ + 'Priority: optional' \ + 'Section: python' \ + "Depends: libpython3.13-stdlib (= ${PYTHON_DEBIAN_VERSION})" \ + "Replaces: libpython3.13-stdlib (<= ${PYTHON_DEBIAN_VERSION})" \ + 'Maintainer: NVIDIA NemoClaw Maintainers' \ + 'Description: NemoClaw HTMLParser security backport for Python 3.13' \ + ' Backports the upstream fix for incremental parsing complexity.' \ + >"${package_root}/DEBIAN/control" + refresh_md5sums "${package_root}" + dpkg-deb --build --root-owner-group \ + "${package_root}" \ + "${output_dir}/nemoclaw-python3.13-htmlparser-fix.deb" + test "$(dpkg-deb -f "${output_dir}/nemoclaw-python3.13-htmlparser-fix.deb" Version)" = \ + "${PYTHON_FIX_VERSION}" +} + +mkdir -p "${output_dir}" +architecture="$(dpkg --print-architecture)" +build_libssh2_package "${architecture}" +build_python_fix_package "${architecture}" diff --git a/scripts/security/patches/libssh2-1.11.1-cve-2026.patch b/scripts/security/patches/libssh2-1.11.1-cve-2026.patch new file mode 100644 index 00000000000..b9d49149ca7 --- /dev/null +++ b/scripts/security/patches/libssh2-1.11.1-cve-2026.patch @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Backports the following verified libssh2 upstream commits to 1.11.1: +# 5e4776146552d898b9c0e1b313cd093fa8dc92d0 +# a2ed82d40964bbc0d64cd717aa0a5a892117d2e6 +# a13bb6c773f0d55ad1628cede57e99803cd898d9 +# 42e33d81577ed4b95d4b4f6f845e5ee8efe5eeb4 +diff --git a/src/openssl.c b/src/openssl.c +index 9af96cb..fb1b28b 100644 +--- a/src/openssl.c ++++ b/src/openssl.c +@@ -1041,11 +1041,13 @@ _libssh2_cipher_crypt(_libssh2_cipher_ctx * ctx, + const int aadlen = (is_aesgcm && IS_FIRST(firstlast)) ? 4 : 0; + /* size of AT, if present */ + const int authenticationtag = IS_LAST(firstlast) ? authlen : 0; +- /* length to encrypt */ +- const int cryptlen = (unsigned int)blocksize - aadlen - authenticationtag; ++ unsigned int cryptlen; /* length to encrypt */ + + (void)algo; + +- assert(blocksize <= sizeof(buf)); +- assert(cryptlen >= 0); ++ if(blocksize > sizeof(buf) || ++ blocksize < (size_t)(aadlen + authenticationtag)) ++ return 1; ++ ++ cryptlen = (unsigned int)blocksize - aadlen - authenticationtag; + + #if LIBSSH2_AES_GCM + /* First block */ +diff --git a/src/publickey.c b/src/publickey.c +index 8517e5a..9a5825f 100644 +--- a/src/publickey.c ++++ b/src/publickey.c +@@ -988,6 +988,13 @@ libssh2_publickey_list_fetch(LIBSSH2_PUBLICKEY *pkey, + } + + if(comment_len) { ++ if(pkey->listFetch_s + comment_len > ++ pkey->listFetch_data + pkey->listFetch_data_len) { ++ _libssh2_error(session, LIBSSH2_ERROR_BUFFER_TOO_SMALL, ++ "ListFetch data too short"); ++ goto err_exit; ++ } ++ + list[keys].num_attrs = 1; + list[keys].attrs = + LIBSSH2_ALLOC(session, +diff --git a/src/sftp.c b/src/sftp.c +index 43f2f93..4c80f18 100644 +--- a/src/sftp.c ++++ b/src/sftp.c +@@ -1278,6 +1278,7 @@ static LIBSSH2_SFTP_HANDLE *sftp_open(LIBSSH2_SFTP *sftp, + "got HANDLE FXOK")); + + LIBSSH2_FREE(session, data); ++ data = NULL; + + /* silly situation, but check for a HANDLE */ + rc = sftp_packet_require(sftp, SSH_FXP_HANDLE, +diff --git a/src/transport.c b/src/transport.c +index b16531b..89f1cf3 100644 +--- a/src/transport.c ++++ b/src/transport.c +@@ -235,13 +235,19 @@ static int transport_fullpacket(LIBSSH2_SESSION *session, + unsigned char *decrypt_buffer; + int blocksize = session->remote.crypt->blocksize; + ++ if(p->total_num < mac_len + 4 + (size_t)blocksize) { ++ LIBSSH2_FREE(session, p->payload); ++ p->payload = NULL; ++ return LIBSSH2_ERROR_DECRYPT; ++ } ++ decrypt_size = (ssize_t)(p->total_num - mac_len - 4); ++ + rc = decrypt(session, p->payload + 4, + first_block, blocksize, FIRST_BLOCK); + if(rc) { + return rc; + } + + /* we need buffer for decrypt */ +- decrypt_size = p->total_num - mac_len - 4; + decrypt_buffer = LIBSSH2_ALLOC(session, decrypt_size); + if(!decrypt_buffer) { + return LIBSSH2_ERROR_ALLOC; diff --git a/scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch b/scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch new file mode 100644 index 00000000000..eab13bee010 --- /dev/null +++ b/scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Backports verified CPython 3.13 commit +# 7933f4bf7131aa4140750f9404f5de0aa2969ced to Debian 3.13.5 and treats an +# empty feed as a no-op so zero-length calls cannot grow the pending buffer. +diff --git a/usr/lib/python3.13/html/parser.py b/usr/lib/python3.13/html/parser.py +index 84a32e8..f25b948 100644 +--- a/usr/lib/python3.13/html/parser.py ++++ b/usr/lib/python3.13/html/parser.py +@@ -115,6 +115,9 @@ class HTMLParser(_markupbase.ParserBase): + self.lasttag = '???' + self.interesting = interesting_normal + self.cdata_elem = None ++ self._pending = [] ++ self._pending_len = 0 ++ self._parse_threshold = 1 + super().reset() + + def feed(self, data): +@@ -123,6 +126,29 @@ class HTMLParser(_markupbase.ParserBase): + Call this as often as you want, with as little or as much text + as you want (may include '\n'). + """ +- self.rawdata = self.rawdata + data +- self.goahead(0) ++ # Accumulate new data in a list and only join and parse it once ++ # enough has piled up. Rescanning an unparsed buffer (e.g. an ++ # unterminated tag) and concatenating onto it on every call would ++ # both be quadratic in the input size. ++ if not data: ++ return ++ self._pending_len += len(data) ++ if self._pending_len < self._parse_threshold: ++ self._pending.append(data) ++ else: ++ if not self._pending: ++ self.rawdata += data ++ else: ++ self._pending.append(data) ++ self.rawdata += ''.join(self._pending) ++ self._pending.clear() ++ self._pending_len = 0 ++ n = len(self.rawdata) ++ self.goahead(0) ++ if len(self.rawdata) < n: ++ # Some data was parsed; resume on the next call. ++ self._parse_threshold = 1 ++ else: ++ # Nothing was parsed; wait until the buffer doubles. ++ self._parse_threshold = len(self.rawdata) + +@@ -129,5 +155,9 @@ class HTMLParser(_markupbase.ParserBase): + def close(self): + """Handle any buffered data.""" ++ if self._pending: ++ self.rawdata += ''.join(self._pending) ++ self._pending.clear() ++ self._pending_len = 0 + self.goahead(1) + + __starttag_text = None diff --git a/src/lib/onboard/base-image.ts b/src/lib/onboard/base-image.ts index b562a5c643e..8234e59336c 100644 --- a/src/lib/onboard/base-image.ts +++ b/src/lib/onboard/base-image.ts @@ -43,7 +43,7 @@ export function openClawBaseImageHasSecurityInventory(imageRef: string): boolean 'test -f "$security_inventory"', 'test ! -L "$security_inventory"', `test "$(stat -c '%u:%g:%a' "$security_inventory")" = "0:0:444"`, - `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" | cmp -s - "$security_inventory"`, + `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" | cmp -s - "$security_inventory"`, `printf '%s\\n' "${OPENCLAW_SECURITY_INVENTORY_PROBE_OK}"`, ].join("; "), ], diff --git a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts index 447e75e1c2b..91b38389606 100644 --- a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts +++ b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts @@ -54,6 +54,8 @@ const CANONICAL_POST_GENERATOR_RUN_SHA256 = new Set([ "1197b99bdb996b37a3e4e386a507dfabcdfb2c26a40b015d617f97208668187d", "5cc53ef9c588470f325c5df8189a2eb1525140d947332ae2c0a80fccb2f36ccb", "83567d1fa0e73bef6a3333383c13ace05e26704964ae6a7a76ee24a2f2be3d7e", + "ca1f7b1cb9dd5d467f806792c4072a84ef1e6402c3e8650b6325b95cc186ccdf", + "7e6a6879382f833f17be02ca7d287685b6afa1c423b1e087b3b05dd677d6e325", ]); function instructionSha256(text: string): string { diff --git a/test/fixtures/security/python3.13-html-parser-fixed.txt b/test/fixtures/security/python3.13-html-parser-fixed.txt new file mode 100644 index 00000000000..faf38754f50 --- /dev/null +++ b/test/fixtures/security/python3.13-html-parser-fixed.txt @@ -0,0 +1,523 @@ +"""A parser for HTML and XHTML.""" + +# This file is based on sgmllib.py, but the API is slightly different. + +# XXX There should be a way to distinguish between PCDATA (parsed +# character data -- the normal case), RCDATA (replaceable character +# data -- only char and entity references and end tags are special) +# and CDATA (character data -- only end tags are special). + + +import re +import _markupbase + +from html import unescape +from html.entities import html5 as html5_entities + + +__all__ = ['HTMLParser'] + +# Regular expressions used for parsing + +interesting_normal = re.compile('[&<]') +incomplete = re.compile('&[a-zA-Z#]') + +entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') +charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') + +starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('') +commentclose = re.compile(r'--\s*>') +# Note: +# 1) if you change tagfind/attrfind remember to update locatestarttagend too; +# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# explode, so don't do it. +# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state +# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') +attrfind_tolerant = re.compile( + r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +locatestarttagend_tolerant = re.compile(r""" + <[a-zA-Z][^\t\n\r\f />\x00]* # tag name + (?:[\s/]* # optional whitespace before attribute name + (?:(?<=['"\s/])[^\s/>][^\s/=>]* # attribute name + (?:\s*=+\s* # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\s]* # bare value + ) + \s* # possibly followed by a space + )?(?:\s|/(?!>))* + )* + )? + \s* # trailing whitespace +""", re.VERBOSE) +endendtag = re.compile('>') +# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between +# ') + +# Character reference processing logic specific to attribute values +# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state +def _replace_attr_charref(match): + ref = match.group(0) + # Numeric / hex char refs must always be unescaped + if ref.startswith('&#'): + return unescape(ref) + # Named character / entity references must only be unescaped + # if they are an exact match, and they are not followed by an equals sign + if not ref.endswith('=') and ref[1:] in html5_entities: + return unescape(ref) + # Otherwise do not unescape + return ref + +def _unescape_attrvalue(s): + return attr_charref.sub(_replace_attr_charref, s) + + +class HTMLParser(_markupbase.ParserBase): + """Find tags and other markup and call handler functions. + + Usage: + p = HTMLParser() + p.feed(data) + ... + p.close() + + Start tags are handled by calling self.handle_starttag() or + self.handle_startendtag(); end tags by self.handle_endtag(). The + data between tags is passed from the parser to the derived class + by calling self.handle_data() with the data as argument (the data + may be split up in arbitrary chunks). If convert_charrefs is + True the character references are converted automatically to the + corresponding Unicode character (and self.handle_data() is no + longer split in chunks), otherwise they are passed by calling + self.handle_entityref() or self.handle_charref() with the string + containing respectively the named or numeric reference as the + argument. + """ + + CDATA_CONTENT_ELEMENTS = ("script", "style") + + def __init__(self, *, convert_charrefs=True): + """Initialize and reset this instance. + + If convert_charrefs is True (the default), all character references + are automatically converted to the corresponding Unicode characters. + """ + super().__init__() + self.convert_charrefs = convert_charrefs + self.reset() + + def reset(self): + """Reset this instance. Loses all unprocessed data.""" + self.rawdata = '' + self.lasttag = '???' + self.interesting = interesting_normal + self.cdata_elem = None + self._pending = [] + self._pending_len = 0 + self._parse_threshold = 1 + super().reset() + + def feed(self, data): + r"""Feed data to the parser. + + Call this as often as you want, with as little or as much text + as you want (may include '\n'). + """ + # Accumulate new data in a list and only join and parse it once + # enough has piled up. Rescanning an unparsed buffer (e.g. an + # unterminated tag) and concatenating onto it on every call would + # both be quadratic in the input size. + if not data: + return + self._pending_len += len(data) + if self._pending_len < self._parse_threshold: + self._pending.append(data) + else: + if not self._pending: + self.rawdata += data + else: + self._pending.append(data) + self.rawdata += ''.join(self._pending) + self._pending.clear() + self._pending_len = 0 + n = len(self.rawdata) + self.goahead(0) + if len(self.rawdata) < n: + # Some data was parsed; resume on the next call. + self._parse_threshold = 1 + else: + # Nothing was parsed; wait until the buffer doubles. + self._parse_threshold = len(self.rawdata) + + def close(self): + """Handle any buffered data.""" + if self._pending: + self.rawdata += ''.join(self._pending) + self._pending.clear() + self._pending_len = 0 + self.goahead(1) + + __starttag_text = None + + def get_starttag_text(self): + """Return full source of start tag: '<...>'.""" + return self.__starttag_text + + def set_cdata_mode(self, elem): + self.cdata_elem = elem.lower() + self.interesting = re.compile(r'' % self.cdata_elem, re.I) + + def clear_cdata_mode(self): + self.interesting = interesting_normal + self.cdata_elem = None + + # Internal -- handle data as far as reasonable. May leave state + # and data to be processed by a subsequent call. If 'end' is + # true, force handling all data as if followed by EOF marker. + def goahead(self, end): + rawdata = self.rawdata + i = 0 + n = len(rawdata) + while i < n: + if self.convert_charrefs and not self.cdata_elem: + j = rawdata.find('<', i) + if j < 0: + # if we can't find the next <, either we are at the end + # or there's more text incoming. If the latter is True, + # we can't pass the text to handle_data in case we have + # a charref cut in half at end. Try to determine if + # this is the case before proceeding by looking for an + # & near the end and see if it's followed by a space or ;. + amppos = rawdata.rfind('&', max(i, n-34)) + if (amppos >= 0 and + not re.compile(r'[\s;]').search(rawdata, amppos)): + break # wait till we get all the text + j = n + else: + match = self.interesting.search(rawdata, i) # < or & + if match: + j = match.start() + else: + if self.cdata_elem: + break + j = n + if i < j: + if self.convert_charrefs and not self.cdata_elem: + self.handle_data(unescape(rawdata[i:j])) + else: + self.handle_data(rawdata[i:j]) + i = self.updatepos(i, j) + if i == n: break + startswith = rawdata.startswith + if startswith('<', i): + if starttagopen.match(rawdata, i): # < + letter + k = self.parse_starttag(i) + elif startswith(" + gtpos = rawdata.find('>', i+9) + if gtpos == -1: + return -1 + self.handle_decl(rawdata[i+2:gtpos]) + return gtpos+1 + else: + return self.parse_bogus_comment(i) + + # Internal -- parse bogus comment, return length or -1 if not terminated + # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state + def parse_bogus_comment(self, i, report=1): + rawdata = self.rawdata + assert rawdata[i:i+2] in ('', i+2) + if pos == -1: + return -1 + if report: + self.handle_comment(rawdata[i+2:pos]) + return pos + 1 + + # Internal -- parse processing instr, return end or -1 if not terminated + def parse_pi(self, i): + rawdata = self.rawdata + assert rawdata[i:i+2] == ' + if not match: + return -1 + j = match.start() + self.handle_pi(rawdata[i+2: j]) + j = match.end() + return j + + # Internal -- handle starttag, return end or -1 if not terminated + def parse_starttag(self, i): + self.__starttag_text = None + endpos = self.check_for_whole_start_tag(i) + if endpos < 0: + return endpos + rawdata = self.rawdata + self.__starttag_text = rawdata[i:endpos] + + # Now parse the data between i+1 and j into a tag and attrs + attrs = [] + match = tagfind_tolerant.match(rawdata, i+1) + assert match, 'unexpected call to parse_starttag()' + k = match.end() + self.lasttag = tag = match.group(1).lower() + while k < endpos: + m = attrfind_tolerant.match(rawdata, k) + if not m: + break + attrname, rest, attrvalue = m.group(1, 2, 3) + if not rest: + attrvalue = None + elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ + attrvalue[:1] == '"' == attrvalue[-1:]: + attrvalue = attrvalue[1:-1] + if attrvalue: + attrvalue = _unescape_attrvalue(attrvalue) + attrs.append((attrname.lower(), attrvalue)) + k = m.end() + + end = rawdata[k:endpos].strip() + if end not in (">", "/>"): + self.handle_data(rawdata[i:endpos]) + return endpos + if end.endswith('/>'): + # XHTML-style empty tag: + self.handle_startendtag(tag, attrs) + else: + self.handle_starttag(tag, attrs) + if tag in self.CDATA_CONTENT_ELEMENTS: + self.set_cdata_mode(tag) + return endpos + + # Internal -- check to see if we have a complete starttag; return end + # or -1 if incomplete. + def check_for_whole_start_tag(self, i): + rawdata = self.rawdata + m = locatestarttagend_tolerant.match(rawdata, i) + if m: + j = m.end() + next = rawdata[j:j+1] + if next == ">": + return j + 1 + if next == "/": + if rawdata.startswith("/>", j): + return j + 2 + if rawdata.startswith("/", j): + # buffer boundary + return -1 + # else bogus input + if j > i: + return j + else: + return i + 1 + if next == "": + # end of input + return -1 + if next in ("abcdefghijklmnopqrstuvwxyz=/" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): + # end of input in or before attribute value, or we have the + # '/' from a '/>' ending + return -1 + if j > i: + return j + else: + return i + 1 + raise AssertionError("we should not get here!") + + # Internal -- parse endtag, return end or -1 if incomplete + def parse_endtag(self, i): + rawdata = self.rawdata + assert rawdata[i:i+2] == " + if not match: + return -1 + gtpos = match.end() + match = endtagfind.match(rawdata, i) # + if not match: + if self.cdata_elem is not None: + self.handle_data(rawdata[i:gtpos]) + return gtpos + # find the name: w3.org/TR/html5/tokenization.html#tag-name-state + namematch = tagfind_tolerant.match(rawdata, i+2) + if not namematch: + # w3.org/TR/html5/tokenization.html#end-tag-open-state + if rawdata[i:i+3] == '': + return i+3 + else: + return self.parse_bogus_comment(i) + tagname = namematch.group(1).lower() + # consume and ignore other stuff between the name and the > + # Note: this is not 100% correct, since we might have things like + # , but looking for > after the name should cover + # most of the cases and is much simpler + gtpos = rawdata.find('>', namematch.end()) + self.handle_endtag(tagname) + return gtpos+1 + + elem = match.group(1).lower() # script or style + if self.cdata_elem is not None: + if elem != self.cdata_elem: + self.handle_data(rawdata[i:gtpos]) + return gtpos + + self.handle_endtag(elem) + self.clear_cdata_mode() + return gtpos + + # Overridable -- finish processing of start+end tag: + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + self.handle_endtag(tag) + + # Overridable -- handle start tag + def handle_starttag(self, tag, attrs): + pass + + # Overridable -- handle end tag + def handle_endtag(self, tag): + pass + + # Overridable -- handle character reference + def handle_charref(self, name): + pass + + # Overridable -- handle entity reference + def handle_entityref(self, name): + pass + + # Overridable -- handle data + def handle_data(self, data): + pass + + # Overridable -- handle comment + def handle_comment(self, data): + pass + + # Overridable -- handle declaration + def handle_decl(self, decl): + pass + + # Overridable -- handle processing instruction + def handle_pi(self, data): + pass + + def unknown_decl(self, data): + pass diff --git a/test/helpers/base-apt-security-functions.ts b/test/helpers/base-apt-security-functions.ts index 11f8ea0fd8e..ade879f8c85 100644 --- a/test/helpers/base-apt-security-functions.ts +++ b/test/helpers/base-apt-security-functions.ts @@ -50,7 +50,7 @@ export function runLoggedDockerShell(command: string, tmp: string, functionDefs: fs.writeFileSync(scriptPath, script, { mode: 0o700 }); return spawnSync("bash", [scriptPath], { encoding: "utf-8", - timeout: 5000, + timeout: 15000, }); } @@ -85,9 +85,9 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ` printf "${architecture}\\n"`, ' elif [[ "$#" -eq 1 && "$1" == "--audit" ]]; then', " return 0", - ' elif [[ "$#" -eq 7 && "$1" == "-i" && "${2##*/}" == "libexpat1.deb" && "${3##*/}" == "libonig5.deb" && "${4##*/}" == "libjq1.deb" && "${5##*/}" == "jq.deb" && "${6##*/}" == "vim-common.deb" && "${7##*/}" == "vim-tiny.deb" ]]; then', + ' elif [[ "$#" -eq 9 && "$1" == "-i" && "${2##*/}" == "libexpat1.deb" && "${3##*/}" == "libonig5.deb" && "${4##*/}" == "libjq1.deb" && "${5##*/}" == "jq.deb" && "${6##*/}" == "vim-common.deb" && "${7##*/}" == "vim-tiny.deb" && "${8##*/}" == "libssh2-1t64.deb" && "${9##*/}" == "nemoclaw-python3.13-htmlparser-fix.deb" ]]; then', ' printf "dpkg-install\\n" >> "$call_log"', - ' [[ -f "$2" && -f "$3" && -f "$4" && -f "$5" && -f "$6" && -f "$7" ]]', + ' [[ -f "$2" && -f "$3" && -f "$4" && -f "$5" && -f "$6" && -f "$7" && -f "$8" && -f "$9" ]]', " else", " return 64", " fi", @@ -102,6 +102,8 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ' libjq1|jq) printf "1.8.2-1" ;;', ' perl) printf "5.40.1-6" ;;', ' vim-common|vim-tiny) printf "2:9.2.0782-1" ;;', + ' libssh2-1t64) printf "1.11.1-1+deb13u1+nemoclaw1" ;;', + ' nemoclaw-python3.13-htmlparser-fix) printf "3.13.5-2+deb13u4+nemoclaw1" ;;', " *) return 64 ;;", " esac", "}", @@ -138,11 +140,12 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ` "${hashes.jq} "*/jq.deb) ;;`, ' "6b063038246492c4a20e0a212c896dde4d5aa9f59d6fb43ff33d10080bc53a39 "*/vim-common.deb) ;;', ` "${hashes.vimTiny} "*/vim-tiny.deb) ;;`, + ' "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7 "*/python3.13/html/parser.py) ;;', " *) return 1 ;;", " esac", " (( count += 1 ))", " done", - ' [[ "$count" -eq 6 ]]', + ' [[ "$count" -eq 6 || "$count" -eq 1 ]]', "}", ].join("\n"), [ @@ -166,7 +169,13 @@ export function baseAptSecurityFunctions(architecture: DebianArchitecture): stri ].join("\n"), [ "python3() {", - ' [[ "$#" -eq 2 && "$1" == "-c" && "$2" == "import pyexpat; assert pyexpat.EXPAT_VERSION == \'expat_2.8.2\', pyexpat.EXPAT_VERSION" ]]', + ' [[ "$#" -eq 2 && "$1" == "-c" ]] || return 64', + ' case "$2" in', + " \"import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION\") ;;", + " \"from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('/usr/lib/python3.13/html/parser.py').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''\") ;;", + " \"import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'\") ;;", + " *) return 64 ;;", + " esac", "}", ].join("\n"), [ diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 51b625dcba0..c84db62246a 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -71,7 +71,7 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [ 'test -f "$security_inventory"', 'test ! -L "$security_inventory"', `test "$(stat -c '%u:%g:%a' "$security_inventory")" = "0:0:444"`, - `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" | cmp -s - "$security_inventory"`, + `printf '%s\\n' "architecture=$arch" "libexpat1=2.8.2-1" "libonig5=6.9.9-1+b1" "libjq1=1.8.2-1" "jq=1.8.2-1" "vim-common=2:9.2.0782-1" "vim-tiny=2:9.2.0782-1" "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1" "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1" | cmp -s - "$security_inventory"`, `printf '%s\\n' "nemoclaw-security-inventory-ok"`, ].join("; "); diff --git a/test/helpers/python-parser-security-fixture.ts b/test/helpers/python-parser-security-fixture.ts new file mode 100644 index 00000000000..01ea3d86e5a --- /dev/null +++ b/test/helpers/python-parser-security-fixture.ts @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +export const FIXED_PARSER_SHA256 = + "4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7"; + +export function htmlParserSecurityProbe(parserPath: string): string { + return `from pathlib import Path; import html.parser; assert Path(html.parser.__file__).resolve() == Path('${parserPath}').resolve(); from html.parser import HTMLParser; p=HTMLParser(); [p.feed('') for _ in range(20000)]; assert p._pending == []; p.feed(''); p.close(); assert p.rawdata == ''`; +} + +const PYEXPAT_PROBE = + "import pyexpat; assert pyexpat.EXPAT_VERSION == 'expat_2.8.2', pyexpat.EXPAT_VERSION"; +const LIBSSH2_PROBE = + "import ctypes; lib=ctypes.CDLL('libssh2.so.1'); lib.libssh2_version.restype=ctypes.c_char_p; assert lib.libssh2_version(0) == b'1.11.1'"; +const FIXED_PARSER_FIXTURE = path.resolve( + import.meta.dirname, + "..", + "fixtures", + "security", + "python3.13-html-parser-fixed.txt", +); + +export function stageFixedParser(tmp: string): { + fixedParser: string; + pythonShim: string; +} { + const parserBytes = fs.readFileSync(FIXED_PARSER_FIXTURE); + const parserHash = createHash("sha256").update(parserBytes).digest("hex"); + if (parserHash !== FIXED_PARSER_SHA256) { + throw new Error(`Fixed parser fixture hash mismatch: ${parserHash}`); + } + + const fixedParser = path.join(tmp, "python3.13", "html", "parser.py"); + const pythonShim = path.join(tmp, "usr-bin", "python3"); + fs.mkdirSync(path.dirname(fixedParser), { recursive: true }); + fs.mkdirSync(path.dirname(pythonShim), { recursive: true }); + fs.writeFileSync(fixedParser, parserBytes); + fs.writeFileSync( + pythonShim, + [ + "#!/usr/bin/env python3", + "import html", + "import importlib.util", + "from importlib.machinery import SourceFileLoader", + "import sys", + `parser_path = ${JSON.stringify(fixedParser)}`, + `html_probe = ${JSON.stringify(htmlParserSecurityProbe(fixedParser))}`, + `accepted_noop_probes = {${JSON.stringify(PYEXPAT_PROBE)}, ${JSON.stringify(LIBSSH2_PROBE)}}`, + "if len(sys.argv) != 3 or sys.argv[1] != '-c':", + " raise SystemExit(64)", + "code = sys.argv[2]", + "if code in accepted_noop_probes:", + " raise SystemExit(0)", + "if code != html_probe:", + " raise SystemExit(64)", + "loader = SourceFileLoader('html.parser', parser_path)", + "spec = importlib.util.spec_from_loader(loader.name, loader)", + "if spec is None:", + " raise SystemExit(65)", + "module = importlib.util.module_from_spec(spec)", + "sys.modules[loader.name] = module", + "loader.exec_module(module)", + "html.parser = module", + "exec(code, {})", + "", + ].join("\n"), + { mode: 0o755 }, + ); + return { fixedParser, pythonShim }; +} + +export function useRealPatchedParser(functionDefinitions: string[], pythonShim: string): string[] { + return [ + ...functionDefinitions.filter((definition) => !definition.startsWith("python3() {")), + `python3() { ${JSON.stringify(pythonShim)} "$@"; }`, + ]; +} diff --git a/test/hermes-share-mount-deps.test.ts b/test/hermes-share-mount-deps.test.ts index ec96be1af0a..132f74a9984 100644 --- a/test/hermes-share-mount-deps.test.ts +++ b/test/hermes-share-mount-deps.test.ts @@ -7,14 +7,21 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { BASE_APT_SECURITY_FUNCTIONS } from "./helpers/base-apt-security-functions"; +import { stageFixedParser, useRealPatchedParser } from "./helpers/python-parser-security-fixture"; const ROOT = path.resolve(import.meta.dirname, ".."); const HERMES_DOCKERFILE_BASE = path.join(ROOT, "agents", "hermes", "Dockerfile.base"); function extractAptInstallCommand(dockerfile: string): string { - const match = dockerfile.match( - /RUN\s+apt-get update\s*&&\s*apt-get install -y --no-install-recommends[\s\S]*?&&\s*rm -rf \/var\/lib\/apt\/lists\/\*/m, + const runtimeStage = dockerfile.indexOf( + "FROM node:24-trixie-slim@sha256:05c08ce4291e9a58f59456a7985176defb12cdd42271f35ff81a3e167ea61d4c", ); + expect(runtimeStage).toBeGreaterThanOrEqual(0); + const match = dockerfile + .slice(runtimeStage) + .match( + /RUN\s+apt-get update\s*&&\s*apt-get install -y --no-install-recommends[\s\S]*?&&\s*rm -rf \/var\/lib\/apt\/lists\/\*/m, + ); expect(match).not.toBeNull(); return match![0].replace(/^RUN\s+/, "").replace(/\\\n/g, " "); } @@ -64,7 +71,7 @@ function runLoggedShell(command: string, tmp: string, functionDefs: string[] = [ command, ].join("\n"); fs.writeFileSync(scriptPath, script, { mode: 0o700 }); - const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000 }); + const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 15000 }); const calls = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf-8") : ""; return { result, calls }; } @@ -191,21 +198,32 @@ describe("Hermes share mount package parity (#2947)", () => { const dockerfile = fs.readFileSync(HERMES_DOCKERFILE_BASE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-share-apt-")); const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); + const debianSecurityDebs = path.join(tmp, "debian-security-debs"); + const nativeSecurityDebs = path.join(tmp, "native-security-debs"); const inventoryDirectory = path.join(tmp, "security-inventory"); const inventory = path.join(inventoryDirectory, "security-packages.txt"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); fs.mkdirSync(lists); + fs.mkdirSync(debianSecurityDebs); + fs.mkdirSync(nativeSecurityDebs); + fs.writeFileSync(path.join(nativeSecurityDebs, "libssh2-1t64.deb"), "fixed libssh2"); + fs.writeFileSync( + path.join(nativeSecurityDebs, "nemoclaw-python3.13-htmlparser-fix.deb"), + "fixed parser package", + ); try { const command = extractAptInstallCommand(dockerfile) .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) + .replaceAll("/tmp/nemoclaw-debian-security", debianSecurityDebs) + .replaceAll("/tmp/nemoclaw-native-security", nativeSecurityDebs) .replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory) - .replaceAll("/usr/local/share/nemoclaw", inventoryDirectory); + .replaceAll("/usr/local/share/nemoclaw", inventoryDirectory) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); const { result, calls } = runLoggedShell(command, tmp, [ 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...BASE_APT_SECURITY_FUNCTIONS, + ...useRealPatchedParser(BASE_APT_SECURITY_FUNCTIONS, pythonShim), ]); expect(result.status).toBe(0); @@ -214,6 +232,8 @@ describe("Hermes share mount package parity (#2947)", () => { expect(calls).toContain("procps=2:4.0.4-9"); expect(calls).toContain("e2fsprogs=1.47.2-3+b11"); expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u4"); + expect(fs.existsSync(debianSecurityDebs)).toBe(false); + expect(fs.existsSync(nativeSecurityDebs)).toBe(false); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/native-security-packages.test.ts b/test/native-security-packages.test.ts new file mode 100644 index 00000000000..f5f7c8f2ab9 --- /dev/null +++ b/test/native-security-packages.test.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const BUILD_SCRIPT = path.join(ROOT, "scripts", "security", "build-native-security-packages.sh"); +const LIBSSH2_PATCH = path.join( + ROOT, + "scripts", + "security", + "patches", + "libssh2-1.11.1-cve-2026.patch", +); +const PYTHON_PATCH = path.join( + ROOT, + "scripts", + "security", + "patches", + "python3.13-htmlparser-cve-2026-15308.patch", +); +const BASE_DOCKERFILES = [ + path.join(ROOT, "Dockerfile.base"), + path.join(ROOT, "agents", "hermes", "Dockerfile.base"), + path.join(ROOT, "agents", "langchain-deepagents-code", "Dockerfile.base"), +] as const; + +describe("native security package remediation", () => { + it("keeps the package builder syntactically valid", () => { + const result = spawnSync("bash", ["-n", BUILD_SCRIPT], { encoding: "utf-8" }); + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); + }); + + it("records every reviewed upstream fix at the patch boundary", () => { + const libssh2Patch = fs.readFileSync(LIBSSH2_PATCH, "utf-8"); + for (const commit of [ + "5e4776146552d898b9c0e1b313cd093fa8dc92d0", + "a2ed82d40964bbc0d64cd717aa0a5a892117d2e6", + "a13bb6c773f0d55ad1628cede57e99803cd898d9", + "42e33d81577ed4b95d4b4f6f845e5ee8efe5eeb4", + ]) { + expect(libssh2Patch).toContain(commit); + } + expect(libssh2Patch).toContain("blocksize > sizeof(buf)"); + expect(libssh2Patch).toContain("pkey->listFetch_s + comment_len"); + expect(libssh2Patch).toContain("data = NULL"); + expect(libssh2Patch).toContain("p->total_num < mac_len + 4 + (size_t)blocksize"); + + const pythonPatch = fs.readFileSync(PYTHON_PATCH, "utf-8"); + expect(pythonPatch).toContain("7933f4bf7131aa4140750f9404f5de0aa2969ced"); + expect(pythonPatch).toContain("if not data:"); + expect(pythonPatch).toContain("self._pending_len += len(data)"); + expect(pythonPatch).toContain("self._parse_threshold = len(self.rawdata)"); + }); + + it.each(BASE_DOCKERFILES)("wires both native packages into %s", (dockerfile) => { + const content = fs.readFileSync(dockerfile, "utf-8"); + expect(content).toContain("AS native-security-builder"); + expect(content).toContain( + "COPY scripts/security/build-native-security-packages.sh /scripts/security/build-native-security-packages.sh", + ); + expect(content).toContain( + "COPY scripts/security/patches/libssh2-1.11.1-cve-2026.patch /scripts/security/patches/libssh2-1.11.1-cve-2026.patch", + ); + expect(content).toContain( + "COPY scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch /scripts/security/patches/python3.13-htmlparser-cve-2026-15308.patch", + ); + expect(content).toContain("bash /scripts/security/build-native-security-packages.sh /out"); + expect(content).toContain("openssh-server=1:10.0p1-7+deb13u4"); + expect(content).toContain("/tmp/nemoclaw-native-security/libssh2-1t64.deb"); + expect(content).toContain( + "/tmp/nemoclaw-native-security/nemoclaw-python3.13-htmlparser-fix.deb", + ); + expect(content).toContain("libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1"); + expect(content).toContain("nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1"); + expect(content).toContain("4ff43a8578bda2f14686c67911b64c18e869841973722b1c623b5727491bdaf7"); + expect(content).toContain("[p.feed('') for _ in range(20000)]"); + expect(content).toContain("lib.libssh2_version(0) == b'1.11.1'"); + }); +}); diff --git a/test/perl-critical-cve-remediation.test.ts b/test/perl-critical-cve-remediation.test.ts index 98f6eade210..466c1a4215a 100644 --- a/test/perl-critical-cve-remediation.test.ts +++ b/test/perl-critical-cve-remediation.test.ts @@ -27,6 +27,22 @@ function completedStage(): string { return dockerfile.slice(start); } +function runInstructionContaining(stage: string, needle: string): string { + const lines = stage.split("\n"); + const needleLine = lines.findIndex((line) => line.includes(needle)); + expect(needleLine, `missing ${needle}`).toBeGreaterThanOrEqual(0); + let start = needleLine; + while (start >= 0 && !lines[start]?.startsWith("RUN ")) { + start -= 1; + } + expect(start, `missing RUN instruction for ${needle}`).toBeGreaterThanOrEqual(0); + let end = start; + while (end < lines.length - 1 && lines[end]?.trimEnd().endsWith("\\")) { + end += 1; + } + return lines.slice(start, end + 1).join("\n"); +} + function argumentDefault(name: string): string | undefined { return dockerfile.match(new RegExp(`^ARG ${name}=([^\\s]+)$`, "mu"))?.[1]; } @@ -123,6 +139,10 @@ describe("sandbox base critical Perl CVE remediation", () => { it("replaces the vulnerable distro packages without breaking dpkg ownership (#7338)", () => { const builder = stageNamed("perl-builder"); const runtime = completedStage(); + const perlInstall = runInstructionContaining( + runtime, + "/tmp/nemoclaw-native-security/perl-base.deb", + ); expect(builder).toContain( '"Provides: libperl5.40 (= ${package_version}), perl-modules-5.40 (= ${package_version})"', @@ -131,9 +151,9 @@ describe("sandbox base critical Perl CVE remediation", () => { expect(builder).toContain( '"Replaces: libperl5.40, perl-modules-5.40, perl (<< ${package_version})"', ); - expect(runtime).toContain( - "apt-get install -y --no-install-recommends /tmp/perl-base.deb /tmp/perl.deb", - ); + expect(perlInstall.match(/apt-get install -y --no-install-recommends/g)).toHaveLength(1); + expect(perlInstall).toContain("/tmp/nemoclaw-native-security/perl-base.deb"); + expect(perlInstall).toContain("/tmp/nemoclaw-native-security/perl.deb"); expect(runtime).toContain('test -z "$(dpkg --audit)"'); }); diff --git a/test/sandbox-base-runtime-tools.test.ts b/test/sandbox-base-runtime-tools.test.ts new file mode 100644 index 00000000000..20ed95a3e55 --- /dev/null +++ b/test/sandbox-base-runtime-tools.test.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + BASE_APT_SECURITY_FUNCTIONS, + dockerRunCommandBetween, + runLoggedDockerShell, +} from "./helpers/base-apt-security-functions"; +import { stageFixedParser, useRealPatchedParser } from "./helpers/python-parser-security-fixture"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const DOCKERFILE_BASE = path.join(ROOT, "Dockerfile.base"); +const fixtures: string[] = []; + +function runBaseAptLayer(prefix: string) { + const source = fs.readFileSync(DOCKERFILE_BASE, "utf-8"); + const completedStage = source.lastIndexOf("\nFROM "); + const dockerfile = completedStage >= 0 ? source.slice(completedStage) : source; + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + fixtures.push(tmp); + const lists = path.join(tmp, "apt-lists"); + const debianSecurityDebs = path.join(tmp, "debian-security-debs"); + const nativeSecurityDebs = path.join(tmp, "native-security-debs"); + const fakePythonLink = path.join(tmp, "usr-local-bin", "python"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); + fs.mkdirSync(lists); + fs.mkdirSync(debianSecurityDebs); + fs.mkdirSync(nativeSecurityDebs); + fs.mkdirSync(path.dirname(fakePythonLink), { recursive: true }); + fs.writeFileSync(path.join(nativeSecurityDebs, "libssh2-1t64.deb"), "fixed libssh2"); + fs.writeFileSync( + path.join(nativeSecurityDebs, "nemoclaw-python3.13-htmlparser-fix.deb"), + "fixed parser package", + ); + const command = dockerRunCommandBetween( + dockerfile, + "RUN apt-get update", + "# gosu for privilege separation", + ) + .replaceAll("/var/lib/apt/lists", lists) + .replaceAll("/tmp/nemoclaw-debian-security", debianSecurityDebs) + .replaceAll("/tmp/nemoclaw-native-security", nativeSecurityDebs) + .replaceAll("/usr/local/share/nemoclaw", path.join(tmp, "security-inventory")) + .replaceAll("/usr/local/bin/python", fakePythonLink) + .replaceAll("/usr/bin/python3", pythonShim) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); + const result = runLoggedDockerShell(command, tmp, [ + 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', + 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', + 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', + ...useRealPatchedParser(BASE_APT_SECURITY_FUNCTIONS, pythonShim), + ]); + const calls = fs.readFileSync(path.join(tmp, "calls.log"), "utf-8"); + return { calls, fakePythonLink, pythonShim, result }; +} + +afterEach(() => { + for (const fixture of fixtures.splice(0)) { + fs.rmSync(fixture, { recursive: true, force: true }); + } +}); + +describe("sandbox base runtime tools", () => { + it("installs the required process, filesystem, and SFTP tools", () => { + const { calls, result } = runBaseAptLayer("nemoclaw-base-apt-"); + + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); + expect(calls).toContain("procps=2:4.0.4-9"); + expect(calls).toContain("e2fsprogs=1.47.2-3+b11"); + expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u4"); + }); + + it("symlinks bare `python` to the tested python3 interpreter (#1452)", () => { + const { fakePythonLink, pythonShim, result } = runBaseAptLayer("nemoclaw-base-pysymlink-"); + + expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); + expect(fs.lstatSync(fakePythonLink).isSymbolicLink()).toBe(true); + expect(fs.readlinkSync(fakePythonLink)).toBe(pythonShim); + }); +}); diff --git a/test/sandbox-base-security-packages.test.ts b/test/sandbox-base-security-packages.test.ts index e4ac64d8991..3c286da244b 100644 --- a/test/sandbox-base-security-packages.test.ts +++ b/test/sandbox-base-security-packages.test.ts @@ -11,6 +11,7 @@ import { dockerRunCommandBetween, runLoggedDockerShell, } from "./helpers/base-apt-security-functions"; +import { stageFixedParser, useRealPatchedParser } from "./helpers/python-parser-security-fixture"; const ROOT = path.resolve(import.meta.dirname, ".."); const SECURITY_IMAGES = [ @@ -44,27 +45,41 @@ const SECURITY_CASES = SECURITY_IMAGES.flatMap((image) => function sandboxSecurityCommand( image: (typeof SECURITY_IMAGES)[number], tmp: string, -): { command: string; inventory: string; securityDebs: string } { +): { + command: string; + inventory: string; + debianSecurityDebs: string; + nativeSecurityDebs: string; + pythonShim: string; +} { const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); + const debianSecurityDebs = path.join(tmp, "debian-security-debs"); + const nativeSecurityDebs = path.join(tmp, "native-security-debs"); const inventoryDirectory = path.join(tmp, "security-inventory"); const inventory = path.join(inventoryDirectory, "security-packages.txt"); - const fakePython3 = path.join(tmp, "usr-bin", "python3"); const fakePythonLink = path.join(tmp, "usr-local-bin", "python"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); fs.mkdirSync(lists); - fs.mkdirSync(path.dirname(fakePython3), { recursive: true }); + fs.mkdirSync(debianSecurityDebs); + fs.mkdirSync(nativeSecurityDebs); fs.mkdirSync(path.dirname(fakePythonLink), { recursive: true }); - fs.writeFileSync(fakePython3, "#!/bin/sh\n", { mode: 0o755 }); + fs.writeFileSync(path.join(nativeSecurityDebs, "libssh2-1t64.deb"), "fixed libssh2"); + fs.writeFileSync( + path.join(nativeSecurityDebs, "nemoclaw-python3.13-htmlparser-fix.deb"), + "fixed parser package", + ); const dockerfile = fs.readFileSync(image.dockerfile, "utf-8"); const command = dockerRunCommandBetween(dockerfile, image.startMarker, image.endMarker) .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) + .replaceAll("/tmp/nemoclaw-debian-security", debianSecurityDebs) + .replaceAll("/tmp/nemoclaw-native-security", nativeSecurityDebs) .replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory) .replaceAll("/usr/local/share/nemoclaw", inventoryDirectory) .replaceAll("/usr/local/bin/python", fakePythonLink) - .replaceAll("/usr/bin/python3", fakePython3); - return { command, inventory, securityDebs }; + .replaceAll("/usr/bin/python3", pythonShim) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); + return { command, inventory, debianSecurityDebs, nativeSecurityDebs, pythonShim }; } function securityInventory(architecture: (typeof ARCHITECTURES)[number]): string { @@ -76,6 +91,8 @@ function securityInventory(architecture: (typeof ARCHITECTURES)[number]): string "jq=1.8.2-1", "vim-common=2:9.2.0782-1", "vim-tiny=2:9.2.0782-1", + "libssh2-1t64=1.11.1-1+deb13u1+nemoclaw1", + "nemoclaw-python3.13-htmlparser-fix=3.13.5-2+deb13u4+nemoclaw1", "", ].join("\n"); } @@ -84,16 +101,19 @@ function completedImageSecurityCommand( image: (typeof SECURITY_IMAGES)[number], tmp: string, architecture: (typeof ARCHITECTURES)[number], -): { command: string; inventory: string } { +): { command: string; inventory: string; pythonShim: string } { const inventory = path.join(tmp, "security-packages.txt"); + const { fixedParser, pythonShim } = stageFixedParser(tmp); fs.writeFileSync(inventory, securityInventory(architecture), { mode: 0o444 }); const dockerfile = fs.readFileSync(image.finalDockerfile, "utf-8"); const command = dockerRunCommandBetween( dockerfile, "# Verify the immutable security package inventory in the completed image.", "# End completed-image security package verification.", - ).replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory); - return { command, inventory }; + ) + .replaceAll("/usr/local/share/nemoclaw/security-packages.txt", inventory) + .replaceAll("/usr/lib/python3.13/html/parser.py", fixedParser); + return { command, inventory, pythonShim }; } describe("sandbox base security packages", () => { @@ -101,20 +121,20 @@ describe("sandbox base security packages", () => { SECURITY_CASES, )("executes the exact security package contract for %s on %s", (_name, architecture, image) => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-security-")); - const { command, inventory, securityDebs } = sandboxSecurityCommand(image, tmp); + const prepared = sandboxSecurityCommand(image, tmp); try { - const result = runLoggedDockerShell(command, tmp, [ + const result = runLoggedDockerShell(prepared.command, tmp, [ 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...baseAptSecurityFunctions(architecture), + ...useRealPatchedParser(baseAptSecurityFunctions(architecture), prepared.pythonShim), ]); expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); const calls = fs.readFileSync(path.join(tmp, "calls.log"), "utf-8"); expect(calls).toContain("dpkg-install"); - expect(fs.readFileSync(inventory, "utf-8")).toBe(securityInventory(architecture)); - expect(fs.statSync(inventory).mode & 0o777).toBe(0o444); + expect(fs.readFileSync(prepared.inventory, "utf-8")).toBe(securityInventory(architecture)); + expect(fs.statSync(prepared.inventory).mode & 0o777).toBe(0o444); expect( calls .split("\n") @@ -128,7 +148,9 @@ describe("sandbox base security packages", () => { "vim-common_9.2.0782-1_all.deb", `vim-tiny_9.2.0782-1_${architecture}.deb`, ]); - expect(fs.existsSync(securityDebs)).toBe(false); + expect(prepared.debianSecurityDebs).not.toBe(prepared.nativeSecurityDebs); + expect(fs.existsSync(prepared.debianSecurityDebs)).toBe(false); + expect(fs.existsSync(prepared.nativeSecurityDebs)).toBe(image.name === "OpenClaw"); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } @@ -148,7 +170,7 @@ describe("sandbox base security packages", () => { ' printf "0:0:444\\n"', "}", ].join("\n"), - ...baseAptSecurityFunctions(architecture), + ...useRealPatchedParser(baseAptSecurityFunctions(architecture), prepared.pythonShim), ]); expect({ status: result.status, stderr: result.stderr }).toEqual({ status: 0, stderr: "" }); } finally { @@ -169,11 +191,12 @@ describe("sandbox base security packages", () => { try { const result = runLoggedDockerShell(command, tmp, [ 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', - ...baseAptSecurityFunctions(architecture), + ...useRealPatchedParser(baseAptSecurityFunctions(architecture), prepared.pythonShim), ]); expect(result.status).not.toBe(0); expect(fs.readFileSync(path.join(tmp, "calls.log"), "utf-8")).not.toContain("dpkg-install"); - expect(fs.existsSync(prepared.securityDebs)).toBe(true); + expect(fs.existsSync(prepared.debianSecurityDebs)).toBe(true); + expect(fs.existsSync(prepared.nativeSecurityDebs)).toBe(true); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index a6d4b5e0c93..b5c15e1a9dc 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -14,7 +14,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { BASE_APT_SECURITY_FUNCTIONS } from "./helpers/base-apt-security-functions"; const ROOT = path.resolve(import.meta.dirname, ".."); const DOCKERFILE = path.join(ROOT, "Dockerfile"); @@ -1051,80 +1050,6 @@ describe("sandbox provisioning: base runtime tools", () => { expect(aptInstall).toBeDefined(); expect(aptInstall).toContain("nftables=1.1.3-1"); }); - it("base apt layer requests procps, e2fsprogs, and the SFTP server", () => { - const dockerfile = completedDockerStage(fs.readFileSync(DOCKERFILE_BASE, "utf-8")); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-apt-")); - const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); - const fakePy3 = path.join(tmp, "usr-bin", "python3"); - const fakePyLink = path.join(tmp, "usr-local-bin", "python"); - fs.mkdirSync(lists); - fs.mkdirSync(path.dirname(fakePy3), { recursive: true }); - fs.mkdirSync(path.dirname(fakePyLink), { recursive: true }); - fs.writeFileSync(fakePy3, "#!/bin/sh\n", { mode: 0o755 }); - const command = dockerRunCommandBetween( - dockerfile, - "RUN apt-get update", - "# gosu for privilege separation", - ) - .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) - .replaceAll("/usr/local/share/nemoclaw", path.join(tmp, "security-inventory")) - .replaceAll("/usr/local/bin/python", fakePyLink) - .replaceAll("/usr/bin/python3", fakePy3); - try { - const { result, calls } = runLoggedDockerShell(command, tmp, [ - 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', - 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', - 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...BASE_APT_SECURITY_FUNCTIONS, - ]); - expect(result.status).toBe(0); - expect(calls).toContain("apt-get update"); - expect(calls).toContain("procps=2:4.0.4-9"); - expect(calls).toContain("e2fsprogs=1.47.2-3+b11"); - expect(calls).toContain("openssh-sftp-server=1:10.0p1-7+deb13u4"); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - - it("symlinks bare `python` to python3 so agent tool calls don't fail with command-not-found (#1452)", () => { - const dockerfile = completedDockerStage(fs.readFileSync(DOCKERFILE_BASE, "utf-8")); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-pysymlink-")); - const lists = path.join(tmp, "apt-lists"); - const securityDebs = path.join(tmp, "security-debs"); - const fakePy3 = path.join(tmp, "usr-bin", "python3"); - const fakePyLink = path.join(tmp, "usr-local-bin", "python"); - fs.mkdirSync(lists, { recursive: true }); - fs.mkdirSync(path.dirname(fakePy3), { recursive: true }); - fs.mkdirSync(path.dirname(fakePyLink), { recursive: true }); - fs.writeFileSync(fakePy3, "#!/bin/sh\necho 3.13\n", { mode: 0o755 }); - const command = dockerRunCommandBetween( - dockerfile, - "RUN apt-get update", - "# gosu for privilege separation", - ) - .replaceAll("/var/lib/apt/lists", lists) - .replaceAll("/tmp/nemoclaw-debian-security", securityDebs) - .replaceAll("/usr/local/share/nemoclaw", path.join(tmp, "security-inventory")) - .replaceAll("/usr/local/bin/python", fakePyLink) - .replaceAll("/usr/bin/python3", fakePy3); - try { - const { result } = runLoggedDockerShell(command, tmp, [ - 'apt-get() { printf "apt-get %s\\n" "$*" >> "$call_log"; }', - 'install() { [[ "$#" -eq 8 && "$1" == "-d" && "$2" == "-o" && "$3" == "root" && "$4" == "-g" && "$5" == "root" && "$6" == "-m" && "$7" == "0755" ]] || return 64; mkdir -p "$8"; }', - 'chown() { [[ "$#" -eq 2 && "$1" == "root:root" ]] || return 64; }', - ...BASE_APT_SECURITY_FUNCTIONS, - ]); - expect(result.status).toBe(0); - expect(fs.lstatSync(fakePyLink).isSymbolicLink()).toBe(true); - expect(fs.readlinkSync(fakePyLink)).toBe(fakePy3); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); - it("runtime hardening installs procps and e2fsprogs when a stale base lacks ps and chattr", () => { const dockerfile = fs.readFileSync(DOCKERFILE, "utf-8"); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-procps-"));