fix(customizer): address CVEs in unsloth and automodel

docker/Dockerfile.nmp-automodel-base

@@ -95,18 +95,52 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     "black>=26.3.1,<27" \
     "GitPython>=3.1.50,<4" \
     "jaraco-context>=6.1.0,<7" \
-    "jupyter-server>=2.18.0,<3" \
+    "jupyter-server>=2.20.0,<3" \
     "jupyterlab>=4.5.7,<5" \
     "mistune>=3.2.1,<4" \
     "notebook>=7.5.6,<8" \
     "onnx>=1.21.0,<2" \
     "pillow>=12.2.0,<13" \
     "pyasn1>=0.6.3,<1" \
+    "pyarrow>=23.0.1,<24" \
+    "msgpack>=1.2.1,<2" \
+    "cryptography>=48.0.1,<49" \
     "tornado>=6.5.5,<7" \
     "urllib3>=2.7.0,<3" \
     "mlflow-skinny>=3.11.1,<3.12.0" \
     "grpcio>=1.81.1,<2" \
-    "wandb==0.27.2"
+    "wandb>=0.28.0,<1"
+
+# <ffmpeg-fix>
+# Bundled FFmpeg in PyAV / opencv wheels is still below nspect 8.1.2 (av 17.1.0 → 8.1.1,
+# opencv-python-headless 4.13.0.92 → 8.0.1). Re-enable these pins once upstream wheels
+# ship FFmpeg >= 8.1.2, then drop the rm -rf cleanup step below.
+#
+# Expected to break without av / opencv (until pins are restored):
+#   - Qwen3-Omni / Qwen2.5-Omni VLM (qwen_omni_utils imports av; qwen3_omni_collate_fn)
+#   - Omni use_audio_in_video (av.open to probe video audio tracks)
+#   - Mistral / Pixtral / Mistral3 image inputs (mistral_common cv2.resize in image tokenizer)
+#   - import albumentations (cv2 required; listed in vlm extra but unused by Automodel today)
+# Still expected to work:
+#   - Qwen VL fine-tune (FORCE_QWENVL_VIDEO_READER=torchcodec; no av import in qwen_vl_utils)
+#   - Qwen3-Omni ASR-only collate (audio/collate_fns avoids qwen_omni_utils)
+#   - Mistral / Pixtral text-only tokenization
+# RUN --mount=type=cache,target=/root/.cache/uv \
+#     uv pip install \
+#     "av>=17.1.0,<18" \
+#     "opencv-python-headless>=4.13.0.92,<4.14"
+
+# Remove PyAV / opencv from the venv so nspect does not scan bundled libav* in av.libs /
+# opencv_python_headless.libs. vlm sync still pulls them transitively (qwen-omni-utils,
+# albumentations, mistral-common[opencv]); see breakage notes above.
+RUN rm -rf \
+    /opt/venv/lib/python*/site-packages/av \
+    /opt/venv/lib/python*/site-packages/av-*.dist-info \
+    /opt/venv/lib/python*/site-packages/av.libs \
+    /opt/venv/lib/python*/site-packages/cv2 \
+    /opt/venv/lib/python*/site-packages/opencv_python_headless.libs \
+    /opt/venv/lib/python*/site-packages/opencv_python_headless-*.dist-info
+# </ffmpeg-fix>
 
 # Published base image (same filesystem as builder).
 FROM ${PYTORCH_BASE} AS nmp-automodel-base
@@ -119,6 +153,12 @@ COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv
 # Stale copies under NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.05-py3).
 # CVE scanners read dist-packages even when --system-site-packages resolves imports from /opt/venv.
 # rm -rf (not pip uninstall) avoids corrupted dist-info RECORDs on the NGC base image.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --only-upgrade openssl libssl3t64 && \
+    rm -rf /var/lib/apt/lists/*
+
+# NGC ships Nsight Systems (~585 MB); not used by nmp-automodel training.
+# Cleanup stale NGC image packages.
 RUN rm -rf \
     /usr/local/lib/python3.12/dist-packages/aiohttp \
     /usr/local/lib/python3.12/dist-packages/aiohttp-*.dist-info \
@@ -141,7 +181,12 @@ RUN rm -rf \
     /usr/local/lib/python3.12/dist-packages/wandb-*.dist-info \
     /usr/local/lib/python3.12/dist-packages/mlflow \
     /usr/local/lib/python3.12/dist-packages/mlflow-*.dist-info \
-    /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info
+    /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info \
+    /usr/local/lib/python3.12/dist-packages/jupyter_server \
+    /usr/local/lib/python3.12/dist-packages/jupyter_server-*.dist-info \
+    /usr/local/cuda/NsightSystems-cli-* \
+    /usr/local/bin/nsys \
+    /usr/local/cuda/bin/nsys
 
 ENV VIRTUAL_ENV=/opt/venv \
     UV_PROJECT_ENVIRONMENT=/opt/venv \

docker/Dockerfile.nmp-automodel-training

@@ -40,7 +40,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache --no-deps \
     -e /opt/Automodel
 
-
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 ENTRYPOINT ["/opt/venv/bin/python"]
 CMD ["-m", "nmp.automodel.tasks.training", "--help"]

docker/Dockerfile.nmp-unsloth-training

@@ -153,7 +153,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     -e /app/packages/nemo_platform_plugin \
     -e /app/packages/nmp_common \
     -e /app/packages/nmp_customization_common \
-    -e "/app/services/unsloth[integrations]"
+    -e "/app/services/unsloth"
 
 # hf-transfer can pull huggingface-hub 1.x — install then re-pin hub + transformers.
 RUN --mount=type=cache,target=/root/.cache/uv \
@@ -179,6 +179,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     "mlflow-skinny>=3.11.1,<3.12.0"
 
 # Stale NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.05-py3).
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --only-upgrade openssl libssl3t64 && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN rm -rf \
     /usr/local/lib/python3.12/dist-packages/aiohttp \
     /usr/local/lib/python3.12/dist-packages/aiohttp-*.dist-info \
@@ -199,7 +203,13 @@ RUN rm -rf \
     /usr/local/lib/python3.12/dist-packages/wandb-*.dist-info \
     /usr/local/lib/python3.12/dist-packages/mlflow \
     /usr/local/lib/python3.12/dist-packages/mlflow-*.dist-info \
-    /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info
+    /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info \
+    /usr/local/lib/python3.12/dist-packages/jupyter_server \
+    /usr/local/lib/python3.12/dist-packages/jupyter_server-*.dist-info \
+    /usr/local/cuda/NsightSystems-cli-*  \
+    /usr/local/bin/nsys \
+    /usr/local/cuda/bin/nsys
+
 
 ENTRYPOINT ["/opt/venv/bin/python"]
 CMD ["-m", "nmp.unsloth.tasks.training", "--help"]

docker/automodel/no_override_requirements.txt

@@ -10,4 +10,4 @@ safetensors; sys_platform == 'never'
 numpy; sys_platform == 'never'
 
 # Keep W&B's bundled wandb-core binary on a build with patched go-git/go-billy.
-wandb==0.27.2
+wandb==0.28.0

docker/scripts/cve-cleanup.sh

@@ -15,8 +15,9 @@ for venv in /app/.venv /opt/venv; do
     fi
 done
 
-# remove nsight cli (unused) - nic_sampler vulnerable
+# remove nsight cli (unused) - nic_sampler vulnerable; ~585 MB on NGC PyTorch 26.05
 rm -rf /usr/local/cuda/NsightSystems-cli-*
+rm -f /usr/local/bin/nsys /usr/local/cuda/bin/nsys
 
 # Remove shadowed system packages causing CVEs.
 # Uses targeted rm -rf instead of pip uninstall because the nemo-rl base image

docker/unsloth/no_override_requirements.txt

@@ -11,4 +11,4 @@ accelerate; sys_platform == 'never'
 safetensors; sys_platform == 'never'
 
 # Keep W&B's bundled wandb-core binary on a build with patched go-git/go-billy.
-wandb==0.27.2
+wandb==0.28.0

pyproject.toml

@@ -264,7 +264,7 @@ constraint-dependencies = [
   "cryptography>=46.0.7",
   "diffusers>=0.38.0",
   "filelock>=3.20.3",
-  "jupyter-server>=2.18.0",
+  "jupyter-server>=2.20.0",
   "jupyterlab>=4.5.7",
   "langchain-community>=0.3.31,<0.4",     # 0.4.x removes the vertexai import path used by ragas 0.4.3
   "langchain-core>=1.4.8",
@@ -276,6 +276,7 @@ constraint-dependencies = [
   "litellm>=1.83.10",
   "lxml>=6.1.0",
   "mistune>=3.2.1",
+  "msgpack>=1.2.1",
   "nbconvert>=7.17.1",
   "nltk>=3.9.4",
   "pillow>=12.2.0",
@@ -320,7 +321,7 @@ override-dependencies = [
   "ujson>=5.12.0",           # High – nmp-api/cpu-tasks/gpu-tasks
   "xgrammar>=0.1.32",        # High – customizer + nmp-gpu-tasks
   "fastmcp>=3.2.0",          # GHSA-vv7q-7jx5-f767 (Critical) + GHSA-rww4-4w9c-7733 (High); overrides vendored sdk/python/nemo-platform <3 constraint
-  "wandb>=0.25.1",           # CVE-2026-33186
+  "wandb>=0.28.0",           # wandb-core Go CVEs (nspect); unsloth + training images
   "click>=8.2.0",            # Below CVEs are based on constraints that garak has
   "langchain-openai>=1.3.3",
   "litellm>=1.83.10",        # CVE-2026-42208 (Critical SQL injection), CVE-2026-40217, and 5 other High CVEs

services/unsloth/pyproject.toml

@@ -13,6 +13,8 @@ dependencies = [
     "pydantic-settings>=2.6.1",
     "httpx>=0.27.0",
     "tenacity>=8.5.0",
+    "wandb>=0.28.0",
+    "mlflow-skinny>=3.11.1,<3.12.0",
 ]
 
 # Heavy ML deps live behind an optional extra. The package itself is
@@ -31,12 +33,6 @@ dependencies = [
 # command (per unsloth's README), so this extra is mainly for non-container
 # callers that want a single-package handle.
 unsloth = ["unsloth[huggingface]"]
-# HF Trainer callbacks for integrations.wandb / integrations.mlflow (report_to).
-# Container image installs this extra; keep out of core deps so compile/API stay lean.
-integrations = [
-    "wandb>=0.25.1",
-    "mlflow-skinny",
-]
 
 [project.scripts]
 # Container entrypoints. Each script matches the automodel name pattern for parity.