build: consolidate image cleanup removals

docker/Dockerfile.nmp-unsloth-training

@@ -56,6 +56,14 @@ ENV PATH="/bin:${PATH}"
 
 RUN dpkg-query -W -f='${binary:Package}\t${Version}\n' > /tmp/base-dpkg-packages.txt
 
+RUN --mount=type=bind,source=docker/scripts/remove-listed-apt-packages.sh,target=/tmp/remove-listed-apt-packages.sh,readonly \
+    --mount=type=bind,source=docker/customizer/removals/apt/nmp-unsloth-training.txt,target=/tmp/apt-package-removals.txt,readonly \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --only-upgrade openssl libssl3t64 && \
+    bash /tmp/remove-listed-apt-packages.sh /tmp/apt-package-removals.txt && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 ENV VIRTUAL_ENV=/opt/venv \
     UV_PROJECT_ENVIRONMENT=/opt/venv \
     UV_LINK_MODE=copy \
@@ -193,11 +201,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     "GitPython>=3.1.58,<4" \
     "mlflow-skinny>=3.11.1,<3.12.0"
 
-# Stale NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.07-py3).
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --only-upgrade openssl libssl3t64 && \
-    rm -rf /var/lib/apt/lists/*
-
 RUN rm -rf \
     /usr/local/lib/python3.12/dist-packages/aiohttp \
     /usr/local/lib/python3.12/dist-packages/aiohttp-*.dist-info \
@@ -250,6 +253,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --output /usr/share/nemo-platform/source-distributions/nmp-unsloth-training \
         --label app-venv
 
+# Late file cleanup for bundled or non-package-managed files.
+RUN --mount=type=bind,source=docker/scripts/remove-listed-files.sh,target=/tmp/remove-listed-files.sh,readonly \
+    --mount=type=bind,source=docker/customizer/removals/files/nmp-unsloth-training.txt,target=/tmp/file-removals.txt,readonly \
+    bash /tmp/remove-listed-files.sh /tmp/file-removals.txt
+
 ENTRYPOINT ["/opt/venv/bin/python"]
 CMD ["-m", "nmp.unsloth.tasks.training", "--help"]
 

docker/automodel/Dockerfile.nmp-automodel-base

@@ -152,8 +152,12 @@ RUN dpkg-query -W -f='${binary:Package}\t${Version}\n' > /tmp/base-dpkg-packages
 
 # CVE scanners read dist-packages even when --system-site-packages resolves imports from /opt/venv.
 # rm -rf (not pip uninstall) avoids corrupted dist-info RECORDs on the NGC base image.
-RUN apt-get update && \
+RUN --mount=type=bind,source=docker/scripts/remove-listed-apt-packages.sh,target=/tmp/remove-listed-apt-packages.sh,readonly \
+    --mount=type=bind,source=docker/customizer/removals/apt/nmp-automodel-base.txt,target=/tmp/apt-package-removals.txt,readonly \
+    apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --only-upgrade openssl libssl3t64 && \
+    bash /tmp/remove-listed-apt-packages.sh /tmp/apt-package-removals.txt && \
+    apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 # NGC ships Nsight Systems (~585 MB); not used by nmp-automodel training.

docker/automodel/Dockerfile.nmp-automodel-training

@@ -54,8 +54,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --baseline-freeze /tmp/base-python-freeze.txt \
         --label platform-glue
 
-# Codec cleanup for release image scans.
-RUN bash /app/docker/scripts/remove-listed-files.sh /app/docker/automodel/codec-file-removals.txt
+# Late file cleanup for bundled or non-package-managed files.
+RUN --mount=type=bind,source=docker/scripts/remove-listed-files.sh,target=/tmp/remove-listed-files.sh,readonly \
+    --mount=type=bind,source=docker/customizer/removals/files/nmp-automodel-training.txt,target=/tmp/file-removals.txt,readonly \
+    bash /tmp/remove-listed-files.sh /tmp/file-removals.txt
 
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 ENTRYPOINT ["/opt/venv/bin/python"]

docker/automodel/Dockerfile.platform-workspace

@@ -13,8 +13,6 @@ COPY docker/automodel/pyproject.workspace.toml pyproject.toml
 # (The docker/ tree moved out of services/automodel/, so it no longer rides in
 # via `COPY services/automodel`; copy it explicitly into the workspace slice.)
 COPY docker/automodel/no_override_requirements.txt docker/automodel/no_override_requirements.txt
-COPY docker/automodel/codec-file-removals.txt docker/automodel/codec-file-removals.txt
-COPY docker/scripts/remove-listed-files.sh docker/scripts/remove-listed-files.sh
 # nemo-platform-sdk hatch build force-includes docs/ from repo root.
 # docs/fern/openapi/openapi.yaml is a symlink to ../../../openapi/openapi.yaml.
 COPY docs docs

docker/customizer/removals/apt/nmp-automodel-base.txt

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Apt packages removed from nmp-automodel-base early in the image build.
+flac
+lame
+libflac12t64
+libmp3lame0
+libogg0
+libopus0
+libsndfile1
+libsox-fmt-base
+libvorbis0a
+libvorbisenc2
+libvorbisfile3
+sox

docker/customizer/removals/apt/nmp-rl-base.txt

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Apt packages removed from nmp-rl-base early in the image build.
+# This image currently has no apt package removals.

docker/customizer/removals/apt/nmp-unsloth-training.txt

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Apt packages removed from nmp-unsloth-training early in the image build.
+flac
+lame
+libflac12t64
+libmp3lame0
+libogg0
+libopus0
+libsndfile1
+libsox-fmt-base
+libvorbis0a
+libvorbisenc2
+libvorbisfile3
+sox

docker/customizer/removals/files/nmp-automodel-training.txt

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Late cleanup for bundled or non-package-managed files in nmp-automodel-training.
+/opt/venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libFLAC-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavcodec-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavfilter-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavformat-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavutil-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libogg-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libopus-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libsndfile-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libswscale-*.so*

docker/rl/codec-file-removals.txt → docker/customizer/removals/files/nmp-rl-training.txt

@@ -1,8 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-# Codec cleanup for release image scans. Remove this file and its
-# Dockerfile call when the upstream image contents no longer need this workaround.
+# Late cleanup for bundled or non-package-managed files in nmp-rl-training.
 /opt/nemo_rl_venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so
 /opt/ray_venvs/*/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so
 /opt/uv_cache/archive-v0/*/_soundfile_data/libsndfile_*.so

docker/customizer/removals/files/nmp-unsloth-training.txt

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Late cleanup for bundled or non-package-managed files in nmp-unsloth-training.
+/opt/venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libFLAC-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavcodec-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavfilter-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavformat-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libavutil-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libogg-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libopus-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libsndfile-*.so*
+/usr/local/lib/python*/dist-packages/nvidia/dali/.libs/libswscale-*.so*

docker/rl/Dockerfile.nmp-rl-base

@@ -56,12 +56,15 @@ RUN dpkg-query -W -f='${binary:Package}\t${Version}\n' > /tmp/base-dpkg-packages
 
 # openssl is the only dpkg package here with an out-of-SLA CVE. Upgraded explicitly rather than via
 # a blanket `apt-get upgrade`, which would also pull newer CUDA packages from the NVIDIA repos.
-RUN apt-get update && \
+RUN --mount=type=bind,source=docker/scripts/remove-listed-apt-packages.sh,target=/tmp/remove-listed-apt-packages.sh,readonly \
+    --mount=type=bind,source=docker/customizer/removals/apt/nmp-rl-base.txt,target=/tmp/apt-package-removals.txt,readonly \
+    apt-get update && \
     apt-get install -y --no-install-recommends \
         jq curl git rsync wget less vim ccache gnupg \
         build-essential ca-certificates \
         libibverbs-dev && \
     apt-get install -y --only-upgrade openssl libssl3t64 && \
+    bash /tmp/remove-listed-apt-packages.sh /tmp/apt-package-removals.txt && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # CMake (CUDA extension builds: transformer-engine, mamba-ssm, ...).

docker/rl/Dockerfile.nmp-rl-training

@@ -76,8 +76,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --baseline-freeze /tmp/base-python-freeze.txt \
         --label platform-glue
 
-# Codec cleanup for release image scans.
-RUN bash /app/docker/scripts/remove-listed-files.sh /app/docker/rl/codec-file-removals.txt
+# Late file cleanup for bundled or non-package-managed files.
+RUN --mount=type=bind,source=docker/scripts/remove-listed-files.sh,target=/tmp/remove-listed-files.sh,readonly \
+    --mount=type=bind,source=docker/customizer/removals/files/nmp-rl-training.txt,target=/tmp/file-removals.txt,readonly \
+    bash /tmp/remove-listed-files.sh /tmp/file-removals.txt
 
 # Runtime uv cache: the base's /opt/uv_cache ships READ-ONLY on purpose - the prefetched venvs
 # symlink their package files into it, so a writable cache would let code running in this container

docker/rl/Dockerfile.platform-workspace

@@ -9,8 +9,6 @@
 FROM scratch AS platform-workspace
 # Reduced workspace file for this partial source tree.
 COPY docker/rl/pyproject.workspace.toml pyproject.toml
-COPY docker/rl/codec-file-removals.txt docker/rl/codec-file-removals.txt
-COPY docker/scripts/remove-listed-files.sh docker/scripts/remove-listed-files.sh
 # nemo-platform-sdk's hatch build force-includes docs/ from the repo root; the
 # openapi symlink must resolve at build time, so copy both trees.
 COPY docs docs

docker/scripts/remove-listed-apt-packages.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "usage: $0 PACKAGE_LIST" >&2
+    exit 2
+fi
+
+package_list="$1"
+if [ ! -f "${package_list}" ]; then
+    echo "missing package list: ${package_list}" >&2
+    exit 1
+fi
+
+packages=()
+
+while IFS= read -r package || [ -n "${package}" ]; do
+    package="${package%$'\r'}"
+    package="${package%%#*}"
+    package="$(printf '%s' "${package}" | xargs)"
+
+    if [ -z "${package}" ]; then
+        continue
+    fi
+
+    if dpkg-query -W -f='${db:Status-Abbrev}' "${package}" 2>/dev/null | grep -q '^ii '; then
+        packages+=("${package}")
+    fi
+done < "${package_list}"
+
+if [ "${#packages[@]}" -eq 0 ]; then
+    exit 0
+fi
+
+DEBIAN_FRONTEND="${DEBIAN_FRONTEND:-noninteractive}" apt-get purge -y "${packages[@]}"

tests/smoke_gpu/python_package_versions.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Shared helpers for image package minimum-version smoke tests."""
+
+from importlib import metadata
+
+from packaging.version import Version
+
+
+def assert_python_package_min_versions(minimum_versions: dict[str, str]) -> None:
+    missing: list[str] = []
+    too_old: list[str] = []
+
+    for distribution, minimum_version in sorted(minimum_versions.items()):
+        try:
+            actual_version = metadata.version(distribution)
+        except metadata.PackageNotFoundError:
+            missing.append(distribution)
+            continue
+
+        if Version(actual_version) < Version(minimum_version):
+            too_old.append(f"{distribution}: expected >= {minimum_version}, got {actual_version}")
+
+    assert missing == [], f"missing expected Python distributions: {missing}"
+    assert too_old == [], "Python distributions below minimum versions: " + "; ".join(too_old)

tests/smoke_gpu/test_customizer_automodel.py

@@ -19,8 +19,20 @@
 from glob import glob
 
 import pytest
+from python_package_versions import assert_python_package_min_versions
 
 SOUNDFILE_LIBSNDFILE_PATTERN = "/opt/venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so"
+MINIMUM_PYTHON_PACKAGE_VERSIONS = {
+    "bitsandbytes": "0.49.2",
+    "mamba-ssm": "2.3.0",
+    "nvidia-dali-cuda130": "2.2.0",
+    "wandb": "0.28.2",
+}
+
+
+@pytest.mark.smoke_nmp_automodel_training
+def test_python_package_min_versions():
+    assert_python_package_min_versions(MINIMUM_PYTHON_PACKAGE_VERSIONS)
 
 
 @pytest.mark.smoke_nmp_automodel_training
@@ -57,4 +69,13 @@ def test_nmp_automodel_training_importable():
 @pytest.mark.smoke_nmp_automodel_training
 def test_soundfile_libsndfile_removed():
     remaining = sorted(glob(SOUNDFILE_LIBSNDFILE_PATTERN))
-    assert remaining == [], f"codec cleanup left scanner-visible libsndfile files: {remaining}"
+    assert remaining == [], f"file cleanup left scanner-visible libsndfile files: {remaining}"
+
+
+@pytest.mark.smoke_nmp_automodel_training
+def test_dali_still_importable_after_file_cleanup():
+    import importlib.metadata
+
+    import nvidia.dali  # noqa: F401
+
+    assert importlib.metadata.version("nvidia-dali-cuda130")

tests/smoke_gpu/test_customizer_tasks.py

@@ -17,6 +17,18 @@
 """
 
 import pytest
+from python_package_versions import assert_python_package_min_versions
+
+MINIMUM_PYTHON_PACKAGE_VERSIONS = {
+    "mamba-ssm": "2.3.0",
+    "transformers": "5.8.1",
+    "wandb": "0.28.2",
+}
+
+
+@pytest.mark.smoke_nmp_customizer_tasks
+def test_python_package_min_versions():
+    assert_python_package_min_versions(MINIMUM_PYTHON_PACKAGE_VERSIONS)
 
 
 @pytest.mark.smoke_nmp_customizer_tasks

tests/smoke_gpu/test_rl_training.py

@@ -30,13 +30,17 @@
 from pathlib import Path
 
 import pytest
+from python_package_versions import assert_python_package_min_versions
 
 RAY_VENVS = Path("/opt/ray_venvs")
 SOUNDFILE_LIBSNDFILE_PATTERNS = (
     "/opt/nemo_rl_venv/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so",
     "/opt/ray_venvs/*/lib/python3.*/site-packages/_soundfile_data/libsndfile_*.so",
     "/opt/uv_cache/archive-v0/*/_soundfile_data/libsndfile_*.so",
 )
+BASE_VENV_MINIMUM_PYTHON_PACKAGE_VERSIONS = {
+    "wandb": "0.28.2",
+}
 
 # Actor FQN -> packages that must import inside that actor's venv.
 #
@@ -121,6 +125,11 @@ def _import_in_venv(venv: Path, module: str) -> subprocess.CompletedProcess:
 # --- base venv: only what genuinely lives there -------------------------------------------------
 
 
+@pytest.mark.smoke_nmp_rl_training
+def test_base_python_package_min_versions():
+    assert_python_package_min_versions(BASE_VENV_MINIMUM_PYTHON_PACKAGE_VERSIONS)
+
+
 @pytest.mark.smoke_nmp_rl_training
 def test_torch_importable():
     """torch is a default dependency, so it is present in the base (driver) venv."""
@@ -227,7 +236,7 @@ def test_worker_venvs_symlink_into_shared_cache():
 @pytest.mark.smoke_nmp_rl_training
 def test_soundfile_libsndfile_removed():
     remaining = sorted(path for pattern in SOUNDFILE_LIBSNDFILE_PATTERNS for path in glob(pattern))
-    assert remaining == [], f"codec cleanup left scanner-visible libsndfile files: {remaining}"
+    assert remaining == [], f"file cleanup left scanner-visible libsndfile files: {remaining}"
 
 
 @pytest.mark.smoke_nmp_rl_training