diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 143406673a8..3647f00e9cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -78,7 +78,7 @@ jobs: make docker-print TARGET=docker-gpu make docker-print TARGET=docker-auditor make docker-print TARGET=docker-python-base - make docker-print TARGET=gpu-wheels + make docker-print TARGET=nmp-automodel-gpu-wheels make docker-print TARGET=nmp-automodel make docker-print TARGET=nmp-unsloth diff --git a/docker-bake.hcl b/docker-bake.hcl index 8ebbc057728..edec3cd3dcd 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -18,7 +18,7 @@ variable "IMAGE_REGISTRY" { default = "my-registry" } -# Registry where pinned base images are published (nmp-python-base, nmp-customizer-tasks-base, etc.) +# Registry where pinned base images are published. # In CI this matches CI_REGISTRY_IMAGE; locally override if needed. variable "BASE_REGISTRY" { default = "my-registry" @@ -36,10 +36,6 @@ variable "PYTHON_DEV_BASE_TARGET" { default = "" } -variable "CUSTOMIZER_TASKS_BASE_CONTEXT" { - default = "" -} - variable "AUTOMODEL_BASE_CONTEXT" { default = "" } @@ -48,10 +44,6 @@ variable "USE_LOCAL_WHEELS" { default = "" } -variable "FAISS_WHEEL_CONTEXT" { - default = "" -} - variable "CAUSAL_CONV1D_WHEEL_CONTEXT" { default = "" } @@ -86,11 +78,6 @@ variable "BASE_TAG_AUTOMODEL" { default = "dbdc61fdb117f48bc9569b4274afe836cf0de030" } -# Pin for nmp-customizer-tasks-base (built by nmp-customizer-tasks-base-builder) -variable "BASE_TAG_CUSTOMIZER_TASKS" { - default = "dbdc61fdb117f48bc9569b4274afe836cf0de030" -} - # The tag for base images if needed variable "WHEELS_TAG" { default = "3fd6986ff173b598446ffac06d9be3f84b482495" @@ -124,14 +111,6 @@ variable "SAFE_SYNTHESIZER_CONTAINER_VARIANT" { default = "cu129" } -variable "FAISS_VERSION" { - default = "v1.9.0" -} - -variable "FAISS_CUDA_ARCHS" { - default = "75;80;86;90" -} - # Versions for the mamba wheel builder. variable "MAMBA_22_COMMIT" { default = "6b32be06d026e170b3fdaf3ae6282c5a6ff57b06" @@ -145,11 +124,6 @@ variable "CAUSAL_CONV1D_VERSION" { default = "v1.5.3" } -function "get_faiss_wheel_image" { - params = [] - result = "${WHEELS_REGISTRY}/faiss-gpu-wheel:${WHEELS_TAG}" -} - function "get_causal_conv1d_wheel_image" { params = [] result = "${WHEELS_REGISTRY}/causal-conv1d-wheel:${WHEELS_TAG}" @@ -182,21 +156,11 @@ function "python_dev_base_target" { result = notequal(PYTHON_DEV_BASE_TARGET, "") ? PYTHON_DEV_BASE_TARGET : notequal(USE_PREBUILT_BASES, "") ? "nmp-python-dev-base" : "nmp-python-dev-base-builder" } -function "customizer_tasks_base_context" { - params = [] - result = notequal(CUSTOMIZER_TASKS_BASE_CONTEXT, "") ? CUSTOMIZER_TASKS_BASE_CONTEXT : notequal(USE_PREBUILT_BASES, "") ? "docker-image://${BASE_REGISTRY}/nmp-customizer-tasks-base:${BASE_TAG_CUSTOMIZER_TASKS}" : "target:nmp-customizer-tasks-base-builder" -} - function "automodel_base_context" { params = [] result = notequal(AUTOMODEL_BASE_CONTEXT, "") ? AUTOMODEL_BASE_CONTEXT : notequal(USE_PREBUILT_BASES, "") ? "docker-image://${BASE_REGISTRY}/nmp-automodel-base:${BASE_TAG_AUTOMODEL}" : "target:nmp-automodel-base-builder" } -function "faiss_wheel_context" { - params = [] - result = notequal(FAISS_WHEEL_CONTEXT, "") ? FAISS_WHEEL_CONTEXT : notequal(USE_LOCAL_WHEELS, "") ? "target:faiss-gpu-wheel" : "docker-image://${get_faiss_wheel_image()}" -} - function "causal_conv1d_wheel_context" { params = [] result = notequal(CAUSAL_CONV1D_WHEEL_CONTEXT, "") ? CAUSAL_CONV1D_WHEEL_CONTEXT : notequal(USE_LOCAL_WHEELS, "") ? "target:causal-conv1d-wheel" : "docker-image://${get_causal_conv1d_wheel_image()}" @@ -259,15 +223,6 @@ group "docker-auditor" { ] } -# Legacy groups for backward compatibility -group "docker-amd64-only" { - targets = [ - "customizer-rl-docker", - "customizer-tasks-docker", - "docker-gpu" - ] -} - group "all-multi-platform" { targets = [ "docker-multi-platform", @@ -277,7 +232,6 @@ group "all-multi-platform" { group "docker-multi-platform" { targets = [ "docker-cpu", - "customizer-megatron-docker", "auditor-tasks-docker", ] } @@ -298,7 +252,7 @@ group "all-arm64" { group "all-amd64" { targets = [ "docker-multi-platform", - "docker-amd64-only", + "docker-gpu", ] } @@ -306,7 +260,6 @@ group "docker" { targets = [ "docker-cpu", "docker-gpu", - "docker-customizer", "docker-auditor", ] } @@ -332,14 +285,6 @@ group "docker-gpu" { ] } -group "gpu-wheels" { - targets = [ - "faiss-gpu-wheel", - "causal-conv1d-wheel", - "mamba-ssm-wheel", - ] -} - group "nmp-automodel-gpu-wheels" { targets = [ "causal-conv1d-wheel", @@ -363,17 +308,6 @@ group "nmp-unsloth" { ] } -# Final Customizer Training Images - WARNING: No not build base CUDA images using shared buildkit -# Only build the final image. All base image builds should happen using arch specific runners. -group "docker-customizer" { - targets = [ - "customizer-rl-docker", - "customizer-rl-smoke-test", - "customizer-tasks-docker", - "customizer-tasks-smoke-test", - ] -} - # Base images for consolidated containers target "nmp-python-base" { target = python_base_target() @@ -561,63 +495,6 @@ target "nmp-cpu-tasks-docker" { platforms = get_platforms() } -# Safe Synthesizer Tasks - GPU-intensive Safe Synthesizer task execution - -# NMP Customizer Tasks - GPU Tasks for Customizer (transformers>=5.0.0, mamba-ssm 2.3.0) - -# Builder for the Customizer Tasks base image -target "nmp-customizer-tasks-base-builder" { - target = "nmp-customizer-tasks-base-builder" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer-tasks" - cache-to = maybe_registry_cache_to("nmp-customizer-tasks-base") - cache-from = maybe_registry_cache_from("nmp-customizer-tasks-base") - tags = base_tags("nmp-customizer-tasks-base") - output = image_output() - contexts = { - causal-conv1d-wheel-image = causal_conv1d_wheel_context() - faiss-wheel-image = faiss_wheel_context() - mamba-ssm-wheel-image = mamba_ssm_wheel_context() - nmp-gpu-base = "target:nmp-gpu-base" - nmp-workspace = "target:nmp-workspace" - } - platforms = ["linux/amd64"] -} - -# The base image for Customizer Tasks -target "nmp-customizer-tasks-base" { - target = "nmp-customizer-tasks-base-builder" - context = "." - contexts = { - causal-conv1d-wheel-image = causal_conv1d_wheel_context() - faiss-wheel-image = faiss_wheel_context() - mamba-ssm-wheel-image = mamba_ssm_wheel_context() - nmp-gpu-base = "target:nmp-gpu-base" - nmp-workspace = "target:nmp-workspace" - } - dockerfile = "docker/Dockerfile.nmp-customizer-tasks" - platforms = get_platforms() -} - - -# FAISS GPU Wheel Builder -# Clones FAISS from GitHub and produces a Python wheel with CUDA support. -# The wheel lives at /faiss/wheels/*.whl inside the image. -target "faiss-gpu-wheel" { - target = "faiss-gpu-wheel" - context = "." - dockerfile = "docker/base/Dockerfile.faiss-gpu-wheel" - cache-to = maybe_registry_cache_to("faiss-gpu-wheel") - cache-from = maybe_registry_cache_from("faiss-gpu-wheel") - tags = wheel_tags("faiss-gpu-wheel") - output = image_output() - args = { - FAISS_VERSION = FAISS_VERSION - CUDA_ARCHS = FAISS_CUDA_ARCHS - } - platforms = get_platforms() -} - # Mamba Wheel Builders # Builds Python wheels for mamba-ssm and causal-conv1d in parallel. # Both only ship source distributions on PyPI; this pre-builds them for @@ -804,133 +681,6 @@ target "buildkit-test" { platforms = get_platforms() } -# Customizer - -# Customizer base targets (used as context injection) -target "customizer-megatron-base" { - target = "customizer-megatron-base" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer" - platforms = get_platforms() -} - -target "customizer-rl-shared-base" { - target = "customizer-rl-shared-base" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer" - # nemo-rl only for amd64 at the moment - platforms = ["linux/amd64"] -} - -target "customizer-combined-base" { - target = "customizer-combined-base" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer" - platforms = ["linux/amd64"] -} - -# Customizer Base Build Targets - -# Customizer Final Images - base injected via contexts -target "customizer-combined-docker" { - target = "customizer-runtime" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer" - contexts = { - nmp-workspace = "target:nmp-workspace" - customizer-base = "target:customizer-combined-base" - } - cache-to = maybe_registry_cache_to("customizer-combined") - cache-from = maybe_registry_cache_from("customizer-combined") - tags = sha_and_maybe_latest_tags("customizer-combined") - output = image_output() - platforms = ["linux/amd64"] -} - -target "customizer-megatron-docker" { - target = "customizer-runtime" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer" - contexts = { - nmp-workspace = "target:nmp-workspace" - customizer-base = "target:customizer-megatron-base" - } - cache-to = maybe_registry_cache_to("customizer-megatron") - cache-from = maybe_registry_cache_from("customizer-megatron") - tags = sha_and_maybe_latest_tags("customizer-megatron") - output = image_output() - platforms = get_platforms() -} - -target "customizer-tasks-smoke-test" { - target = "smoke-test" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer-tasks" - contexts = { - nmp-customizer-tasks-base = customizer_tasks_base_context() - causal-conv1d-wheel-image = causal_conv1d_wheel_context() - faiss-wheel-image = faiss_wheel_context() - mamba-ssm-wheel-image = mamba_ssm_wheel_context() - nmp-gpu-base = "target:nmp-gpu-base" - nmp-workspace = "target:nmp-workspace" - } - cache-from = maybe_registry_cache_from("customizer-tasks") - output = ["type=cacheonly"] - platforms = ["linux/amd64"] -} - -target "customizer-tasks-docker" { - target = "runtime" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer-tasks" - contexts = { - nmp-customizer-tasks-base = customizer_tasks_base_context() - causal-conv1d-wheel-image = causal_conv1d_wheel_context() - faiss-wheel-image = faiss_wheel_context() - mamba-ssm-wheel-image = mamba_ssm_wheel_context() - nmp-gpu-base = "target:nmp-gpu-base" - nmp-workspace = "target:nmp-workspace" - } - cache-to = maybe_registry_cache_to("customizer-tasks") - cache-from = maybe_registry_cache_from("customizer-tasks") - tags = sha_and_maybe_latest_tags("customizer-tasks") - output = image_output() - platforms = ["linux/amd64"] -} - -target "customizer-rl-docker" { - target = "customizer-runtime" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer" - contexts = { - nmp-workspace = "target:nmp-workspace" - customizer-base = "target:customizer-rl-shared-base" - } - cache-to = maybe_registry_cache_to("customizer-rl") - cache-from = maybe_registry_cache_from("customizer-rl") - tags = sha_and_maybe_latest_tags("customizer-rl") - output = image_output() - # only amd64 for now, until the next release of nemo-rl - platforms = ["linux/amd64"] -} - -target "customizer-rl-smoke-test" { - target = "smoke-test" - context = "." - dockerfile = "docker/Dockerfile.nmp-customizer" - contexts = { - nmp-workspace = "target:nmp-workspace" - customizer-base = "target:customizer-rl-shared-base" - } - args = { - SMOKE_MARKER = "smoke_customizer_rl" - } - cache-from = maybe_registry_cache_from("customizer-rl") - output = ["type=cacheonly"] - platforms = ["linux/amd64"] -} - - # Automodel and Unsloth target "automodel-platform-workspace" { diff --git a/docker/Dockerfile.nmp-customizer b/docker/Dockerfile.nmp-customizer deleted file mode 100644 index 4aec1fdca11..00000000000 --- a/docker/Dockerfile.nmp-customizer +++ /dev/null @@ -1,213 +0,0 @@ -####### -# Legacy Customizer runtime Dockerfile. -# -# This file keeps the non-Automodel customizer images available while the team -# decides whether to retire them: -# - combined - includes nemo-rl plus Automodel/Megatron extras for comparison -# - rl - includes nemo-rl v0.4.0 -# - megatron - derived from nvidia/nemo:25.11.01 -# -# The old customizer-automodel image and base builder are intentionally not -# defined here. -####### - -ARG SMOKE_MARKER=smoke_customizer_rl - -# BASE IMAGES - -#customizer-rl-shared-base: -FROM nvcr.io/nvidia/nemo-rl:v0.4.0 AS customizer-rl-shared-base - -# setup virtual environment properly -ENV VIRTUAL_ENV=/opt/nemo_rl_venv \ - NO_OVERRIDE_REQUIREMENTS_PATH=/app/services/customizer/src/nmp/customizer/tasks/training/backends/nemo_rl/no_override_requirements.txt - -# nvcr.io/nvidia/nemo-rl:v0.4.0's VENV is built with root and the .cache and .local are unavailable... -RUN chmod 755 /root /root/.cache /root/.cache/uv /root/.local /root/.local/share /root/.local/share/uv - -# CVE fix: dirmngr (CVE-2025-68973): upgrade to 2.2.27-3ubuntu2.5+ -# CVE-2025-68973 (High, out-of-SLA): Upgrade gnupg packages. -# --only-upgrade: upgrades installed packages only; skips absent ones silently. -# Remove unused nsight cli which is out of date -RUN apt-get update && apt-get install -y --only-upgrade \ - gnupg gnupg2 gpg gpg-agent gpgconf gpgsm \ - dirmngr gnupg-utils gpgv \ - && dpkg-query -f '${binary:Package}\n' -W 'nsight*' | xargs -r apt-get purge -y \ - && apt-get autoremove \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# Fix CVE security vulnerabilities in pre-installed packages. -# NOTE: These overrides are done here in the Dockerfile rather than in pyproject.toml -# because the vulnerable packages come pre-installed in the NGC base image venv -# (nvcr.io/nvidia/nemo-rl:v0.4.0), not from our uv workspace. The uv workspace's -# override-dependencies only affect packages resolved by our own services; they have -# no effect on packages already baked into the base image's virtual environment. -# --no-deps is essential here for two reasons: -# 1. The nemo-rl base image ships several packages with corrupted dist-info -# (missing METADATA files). Dependency resolution reads all installed -# package metadata, so it fails before installing anything. --no-deps -# skips resolution entirely. -# 2. Without --no-deps, uv would pull transitive dependencies and potentially -# replace NGC-optimized builds of torch, numpy, etc. with generic PyPI -# wheels, breaking GPU functionality. --no-deps installs only the listed -# packages, leaving all other installed packages untouched. -RUN uv pip install --upgrade setuptools && \ - uv pip install --no-deps --no-cache \ - 'anyio>=4.10.0' \ - "urllib3>=2.6.3" \ - "starlette>=0.49.1" \ - "protobuf>=5.29.6,<6.0.dev0" \ - "cryptography>=46.0.5" \ - "python-multipart>=0.0.20" \ - "aiohttp>=3.13.3" \ - "cbor2>=5.9.0" \ - "pyasn1>=0.6.3" \ - "xgrammar>=0.1.32" \ - "Pillow>=12.1.1" \ - "wheel>=0.46.2" \ - "nltk>=3.9.3" \ - 'mlflow>=3.8.0rc0' \ - 'filelock>=3.25.0' - -# ray CVE fix (GHSA-6wgj-66m2-xxp2, GHSA-gx77-xgc2-4888, GHSA-q279-jhrf-cc6v): -# ray 2.52+ restructured internal modules (ray._private to ray._common). A simple -# overlay leaves remnant files from the old version. We must: -# 1. Purge the old ray package directory and dist-info entirely -# 2. Remove pre-built worker venvs (/opt/ray_venvs) so NeMo-RL rebuilds them -# with the new ray version at runtime -# 3. Install the new ray wheel cleanly -RUN set -e && \ - rm -rf $VIRTUAL_ENV/lib/python3.12/site-packages/ray \ - $VIRTUAL_ENV/lib/python3.12/site-packages/ray-*.dist-info && \ - uv pip install --no-deps --no-cache 'ray>=2.52.0' && \ - for venv_dir in /opt/ray_venvs/*/; do \ - [ -d "$venv_dir/lib" ] || continue; \ - rm -rf "$venv_dir"/lib/python*/site-packages/ray \ - "$venv_dir"/lib/python*/site-packages/ray-*.dist-info; \ - uv pip install --no-deps --no-cache --python "$venv_dir/bin/python" 'ray>=2.52.0'; \ - done && \ - python3 -c "import ray; print(f'ray {ray.__version__} at {ray.__file__}'); from ray._common.ray_constants import LOGGING_ROTATE_BYTES; print('ray._common OK')" - -# vllm CVE fix (GHSA-4r2x-xpjr-7cvv, GHSA-3f6c-7fw2-ppm4, GHSA-7972-pg2x-xr59, -# GHSA-8fr4-5q9j-m8gm, GHSA-pmqf-x6x8-p7qw, GHSA-qh4c-xf7m-gxfc, -# GHSA-wr9h-g72x-mwhm, GHSA-79j6-g2m3-jgfw, GHSA-rxc4-3w6r-4v47): -# vllm is not used by the customizer's current training paths (DPO, SFT, LoRA). -# It ships in the nemo-rl base image for generation/rollout (GRPO) which is not -# yet implemented in v2. Removing it eliminates the CVEs without an upgrade. -RUN rm -rf $VIRTUAL_ENV/lib/python3.12/site-packages/vllm \ - $VIRTUAL_ENV/lib/python3.12/site-packages/vllm-*.dist-info && \ - for venv_dir in /opt/ray_venvs/*/; do \ - [ -d "$venv_dir/lib" ] || continue; \ - rm -rf "$venv_dir"/lib/python*/site-packages/vllm \ - "$venv_dir"/lib/python*/site-packages/vllm-*.dist-info; \ - done - -#customizer-combined-base: -FROM customizer-rl-shared-base AS customizer-combined-base -# add megatron and automodel, not installed by default -RUN --mount=type=cache,target=/root/.cache/uv \ - cd /opt/nemo-rl && UV_HTTP_TIMEOUT=120 uv sync --frozen --extra automodel --extra mcore - -#customizer-megatron-base -FROM nvcr.io/nvidia/nemo:25.11.01 AS customizer-megatron-base -# nemo contains most recent version of megatron -ENV VIRTUAL_ENV=/opt/venv - -# CVE fix: dirmngr (CVE-2025-68973): upgrade to 2.2.27-3ubuntu2.5+ -# CVE-2025-68973 (High, out-of-SLA): Upgrade gnupg packages. -# --only-upgrade: upgrades installed packages only; skips absent ones silently. -# Remove unused nsight cli which is out of date -# Remove vllm which is not used (remediation) -RUN apt-get update && apt-get install -y --only-upgrade \ - gnupg gnupg2 gpg gpg-agent gpgconf gpgsm \ - dirmngr gnupg-utils gpgv \ - && dpkg-query -f '${binary:Package}\n' -W 'nsight*' | xargs -r apt-get purge -y \ - && apt-get autoremove \ - && apt-get clean && rm -rf /var/lib/apt/lists/* && \ - rm -rf /usr/local/lib/python*/site-packages/vllm \ - /usr/local/lib/python*/site-packages/vllm-*.dist-info - -# Fix CVE security vulnerabilities in pre-installed packages. -# NOTE: These overrides are done here in the Dockerfile rather than in pyproject.toml -# because the vulnerable packages come pre-installed in the NGC base image venv -# (nvcr.io/nvidia/nemo:25.11.01), not from our uv workspace. The uv workspace's -# override-dependencies only affect packages resolved by our own services; they have -# no effect on packages already baked into the base image's virtual environment. -# nemo-toolkit is intentionally NOT upgraded here: it is tightly coupled to the -# NGC PyTorch build (nvcr.io/nvidia/nemo:25.11.01 uses a custom PyTorch 2.9.0a0 -# with CUDA 13.0 patches). Upgrading could silently replace the NGC build with a -# PyPI wheel lacking CUDA 13.0 support and break distributed training. -# Track via GHSA-9379-mwvr-7wxx/GHSA-hvjw-vp7g-39h5 (nemo-toolkit). -RUN uv pip install --no-cache --python $VIRTUAL_ENV/bin/python \ - "urllib3>=2.6.3" \ - "black>=26.3.1" \ - "cbor2>=5.9.0" \ - "pyasn1>=0.6.3" \ - "aiohttp>=3.13.3" \ - "xgrammar>=0.1.32" \ - "Pillow>=12.1.1" \ - "nbconvert>=7.17.0" \ - 'nemo-toolkit>=2.6.1' \ - 'tornado>=6.5.5' \ - "onnx>=1.21.0" -# onnx: CVE-2026-28500 has no upstream patch yet; track for future upgrade - -## FINAL RUNTIME IMAGE -FROM customizer-base AS customizer-runtime - -# Default environment variables -ENV GUNICORN_WORKERS=1 \ - PORT=8000 \ - HAYSTACK_TELEMETRY_ENABLED=False \ - HF_HUB_ENABLE_HF_TRANSFER=1 \ - OTEL_PYTHON_EXCLUDED_URLS="health" - -ARG USERNAME=ubuntu -ARG USER_UID=1000 -ARG USER_GID=$USER_UID -COPY --chown=${USER_UID}:${USER_GID} --from=nmp-workspace / /app -ENV USERNAME=${USERNAME} \ - HOME=/home/${USERNAME} - -# Set up user home directory -RUN mkdir -p /home/$USERNAME/.cache \ - && ln -s /home/ubuntu /home/customizer \ - && chown -R $USERNAME /home/$USERNAME/ /home/customizer - -# Copy built venv and configure runtime -# If NO_OVERRIDE_REQUIREMENTS_PATH is set, add --overrides $NO_OVERRIDE_REQUIREMENTS_PATH to the uv pip install command -# NO_OVERRIDE_REQUIREMENTS_PATH is a requirements.txt formatted file that lists packages that should not be overridden -# when installing packages into the target $VIRTUAL_ENV -RUN uv pip install --python $VIRTUAL_ENV/bin/python --no-cache \ - ${NO_OVERRIDE_REQUIREMENTS_PATH:+--overrides $NO_OVERRIDE_REQUIREMENTS_PATH} \ - -e /app/services/customizer - -# CVE fix: python-multipart and black for all customizer images. -RUN uv pip install --python $VIRTUAL_ENV/bin/python --upgrade setuptools && \ - uv pip install --python $VIRTUAL_ENV/bin/python --no-cache \ - "python-multipart>=0.0.22" \ - "black>=26.3.1" - -COPY docker/scripts/cve-cleanup.sh /bin/ -RUN bash /bin/cve-cleanup.sh - -ENV LD_LIBRARY_PATH=/opt/amazon-efa-ofi/ofi/lib:/opt/amazon-efa-ofi/efa/lib:/opt/amazon/aws-ofi-nccl/lib:${LD_LIBRARY_PATH} - -USER ${USER_UID}:${USER_GID} - -# ============================================================================= -# Smoke test: verify critical imports resolve correctly. -# Catches two failure classes without requiring GPU hardware: -# - ModuleNotFoundError : package missing from the image -# - ImportError : CUDA extension .so has an undefined symbol -# (ABI mismatch). Surfaces at .so load time before -# any GPU device is needed. -# SMOKE_MARKER selects which test suite runs. -# Built as part of the docker-customizer bake group but never pushed. -# ============================================================================= -FROM customizer-runtime AS smoke-test -ARG SMOKE_MARKER -USER 0 -COPY tests/smoke_gpu/ /smoke_test/ -RUN uv pip install --python $VIRTUAL_ENV/bin/python --no-cache --reinstall pytest && \ - $VIRTUAL_ENV/bin/pytest /smoke_test/ -m ${SMOKE_MARKER} -v diff --git a/docker/Dockerfile.nmp-customizer-tasks b/docker/Dockerfile.nmp-customizer-tasks deleted file mode 100644 index 95ca16cc7f1..00000000000 --- a/docker/Dockerfile.nmp-customizer-tasks +++ /dev/null @@ -1,141 +0,0 @@ -# syntax=docker/dockerfile:1 -# NMP Customizer Tasks - GPU Tasks for Customizer -# Requirements: -# - nemo-platform -# - python 3.11 -# - mamba-ssm 2.3.0 (compatible with nemotron v3) -# - causal-conv1d - -# ============================================================================= -# Base Image -# ============================================================================= -# Supplied by bake. Local builds use the base builder target; CI can override -# the context to a pinned docker-image:// reference. -FROM nmp-customizer-tasks-base AS customizer-tasks-base - -# ============================================================================= -# FAISS wheel source (ARM64 only) -# ============================================================================= -# faiss-gpu-cu12 has no ARM64 wheel on PyPI. The bake file supplies this -# context from a wheel image or from the local faiss-gpu-wheel target. -FROM faiss-wheel-image AS faiss-wheel-src - -# ============================================================================= -# Wheel sources (causal-conv1d + mamba-ssm) -# ============================================================================= -# Pre-built wheels from the gpu-wheels pipeline. -# Both packages only ship source distributions on PyPI and require nvcc to -# compile. Building them inline is slow; instead consume pre-built wheels -# published by the gpu-wheels CI pipeline. -FROM causal-conv1d-wheel-image AS causal-conv1d-wheel-src -FROM mamba-ssm-wheel-image AS mamba-ssm-wheel-src - -# ============================================================================= -# Builder: Install all dependencies -# ============================================================================= -FROM nmp-gpu-base AS nmp-customizer-tasks-base-builder -COPY --from=nmp-workspace . . - -# ensuring we build using the same base as the runtime image, and with the currently configured dependencies -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --group gpu-tasks --extra cu128 --no-editable - -# Install locally-built FAISS wheel on ARM64. -# faiss-gpu-cu12 is excluded by platform marker on aarch64; the equivalent -# FAISS GPU wheel is provided by the faiss-wheel-src context. -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=from=faiss-wheel-src,target=/tmp/faiss-wheel-src \ - if ls /tmp/faiss-wheel-src/wheels/*.whl >/dev/null 2>&1; then \ - uv pip install --no-deps /tmp/faiss-wheel-src/wheels/*.whl; \ - fi - -# Install pre-built CUDA extension wheels (causal-conv1d, mamba-ssm) -# Wheels are built by the gpu-wheels pipeline. Update WHEELS_TAG in docker-bake.hcl -# when new wheels are published. -# Install mamba_ssm with --no-deps to avoid replacing the pinned torch/CUDA stack, -# then add its lightweight Python runtime dependency explicitly. -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=from=causal-conv1d-wheel-src,target=/tmp/causal-conv1d-wheel-src,readonly \ - --mount=from=mamba-ssm-wheel-src,target=/tmp/mamba-ssm-wheel-src,readonly \ - uv pip install --no-deps \ - /tmp/causal-conv1d-wheel-src/wheels/causal_conv1d-*cp311*.whl \ - /tmp/mamba-ssm-wheel-src/wheels/mamba_ssm-2.3.0-cp311*.whl && \ - uv pip install --no-deps "einops>=0.8,<0.9" - -# ============================================================================= -# Runtime: Copy packages in separate layers -# ============================================================================= -FROM nmp-gpu-base AS runtime -WORKDIR /app - -# Layer 1: NVIDIA CUDA libraries (~4.3GB) -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/nvidia \ - /app/.venv/lib/python3.11/site-packages/nvidia - -# Layer 2: PyTorch (~2GB) -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/torch \ - /app/.venv/lib/python3.11/site-packages/torch -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/torchgen \ - /app/.venv/lib/python3.11/site-packages/torchgen -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/functorch \ - /app/.venv/lib/python3.11/site-packages/functorch -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/torch-*.dist-info \ - /app/.venv/lib/python3.11/site-packages/ - -# Layer 3: Inference engines (~3.4GB) -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/flashinfer_cubin \ - /app/.venv/lib/python3.11/site-packages/flashinfer_cubin -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/triton \ - /app/.venv/lib/python3.11/site-packages/triton - -#TODO needs to be built for arm64 -COPY --from=customizer-tasks-base /app/.venv/lib/python3.11/site-packages/xformers \ - /app/.venv/lib/python3.11/site-packages/xformers - -# Layer 4: CUDA Extensions (causal-conv1d, mamba-ssm) -COPY --from=nmp-customizer-tasks-base-builder /app/.venv/lib/python3.11/site-packages/causal_conv1d \ - /app/.venv/lib/python3.11/site-packages/causal_conv1d -COPY --from=nmp-customizer-tasks-base-builder /app/.venv/lib/python3.11/site-packages/mamba_ssm \ - /app/.venv/lib/python3.11/site-packages/mamba_ssm - -# Layer 5: Everything else (venv structure, bin/, dist-info, other packages) -RUN --mount=from=nmp-customizer-tasks-base-builder,source=/app/.venv,target=/mnt/venv,readonly \ - cd /mnt/venv && \ - tar --exclude='lib/python3.11/site-packages/nvidia' \ - --exclude='lib/python3.11/site-packages/torch' \ - --exclude='lib/python3.11/site-packages/torchgen' \ - --exclude='lib/python3.11/site-packages/functorch' \ - --exclude='lib/python3.11/site-packages/torch-*.dist-info' \ - --exclude='lib/python3.11/site-packages/flashinfer_cubin' \ - --exclude='lib/python3.11/site-packages/vllm' \ - --exclude='lib/python3.11/site-packages/triton' \ - --exclude='lib/python3.11/site-packages/xformers' \ - -cf - . | tar -xf - -C /app/.venv - -RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install "transformers>=5.0.0,<5.1.0" - -# setuptools: GHSA-58pv-8j8x-9vj2 -RUN uv pip install --upgrade setuptools - -COPY docker/scripts/cve-cleanup.sh /bin/ -RUN bash /bin/cve-cleanup.sh - -ENV PATH="/app/.venv/bin:$PATH" -ENTRYPOINT ["nemo-platform"] -# Override CMD with "run" in orchestration (e.g., Helm, docker-compose) -CMD ["--help"] - -# ============================================================================= -# Smoke test: verify all customizer task imports resolve correctly. -# Catches two failure classes without requiring GPU hardware: -# - ModuleNotFoundError : package missing from the image -# - ImportError : CUDA extension .so has an undefined symbol -# (ABI mismatch). Surfaces at .so load time before -# any GPU device is needed. -# Built as part of the docker-customizer bake group but never pushed. -# ============================================================================= -FROM runtime AS smoke-test -COPY tests/smoke_gpu/ /smoke_test/ -RUN uv pip install --no-cache --reinstall pytest && \ - pytest /smoke_test/ -m smoke_customizer_tasks -v diff --git a/docker/README.md b/docker/README.md index 11c1bda906b..203f62cc651 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,5 +8,5 @@ Examples: ```bash docker buildx bake --print docker-cpu docker buildx bake --print nmp-automodel -docker buildx bake --print docker-customizer +docker buildx bake --print nmp-unsloth ``` diff --git a/docker/base/Dockerfile.faiss-gpu-wheel b/docker/base/Dockerfile.faiss-gpu-wheel deleted file mode 100644 index 2a3e1e8b380..00000000000 --- a/docker/base/Dockerfile.faiss-gpu-wheel +++ /dev/null @@ -1,95 +0,0 @@ -# syntax=docker/dockerfile:1 -####### -# FAISS GPU Wheel Builder -# -# Builds the FAISS Python wheel with CUDA support. -# The resulting image stores the wheel at /wheels/*.whl. -# -# Downstream images can install the wheel with: -# COPY --from=faiss-gpu-wheel /wheels/*.whl /tmp/wheels/ -# RUN pip install /tmp/wheels/*.whl -# -# Build args: -# FAISS_VERSION - git tag/branch/SHA to clone (default: v1.9.0) -# CUDA_ARCHS - semicolon-separated SM targets (default: 75;80;86;90) -# CUDA_VERSION - CUDA devel image version (default: 12.8.1) -# -# CUDA_ARCHS reference: -# 75 = Turing (T4, RTX 2080) -# 80 = Ampere (A100) -# 86 = Ampere (A10G, RTX 3080/3090) -# 87 = Ampere (Jetson AGX Orin) -# 89 = Ada (RTX 4090) -# 90 = Hopper (H100) -####### - -ARG CUDA_VERSION=12.8.1 - -FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 AS faiss-gpu-wheel-builder - -ARG FAISS_VERSION=v1.9.0 -ARG CUDA_ARCHS="75;80;86;90" - -ENV DEBIAN_FRONTEND=noninteractive - -# Install build dependencies. -# python3.11 + python3.11-dev are available in ubuntu 22.04 universe. -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - git \ - libgflags-dev \ - libopenblas-dev \ - python3.11 \ - python3.11-dev \ - python3.11-venv \ - swig \ - && rm -rf /var/lib/apt/lists/* - -# Install uv and create a Python 3.11 venv with build-time deps. -# Ubuntu/Debian disables ensurepip for the system Python; uv manages its -# own Python toolchain and doesn't depend on pip or ensurepip. -# cmake is installed via PyPI to get a version >= 3.24 (Ubuntu 22.04 ships 3.22). -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv - -RUN uv venv --python 3.11 /opt/venv \ - && uv pip install --python /opt/venv/bin/python3 \ - "cmake>=3.24" ninja numpy packaging setuptools wheel - -ENV PATH="/opt/venv/bin:$PATH" - -# Clone FAISS at the requested version. -RUN git clone --depth 1 --branch ${FAISS_VERSION} \ - https://github.com/facebookresearch/faiss.git /faiss/src - -# Configure. -RUN cmake -B /faiss/build \ - -DFAISS_ENABLE_GPU=ON \ - -DFAISS_ENABLE_RAFT=OFF \ - -DFAISS_ENABLE_PYTHON=ON \ - -DFAISS_ENABLE_MKL=OFF \ - -DFAISS_ENABLE_EXTRAS=OFF \ - -DBUILD_TESTING=OFF \ - -DBLA_VENDOR=OpenBLAS \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \ - -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/g++ \ - -DPython_EXECUTABLE=/opt/venv/bin/python3 \ - /faiss/src - -# Build CPU library, GPU library, and Python bindings. -RUN cmake --build /faiss/build --target faiss faiss_gpu swigfaiss \ - --parallel $(nproc) - -# Package into a wheel. -RUN cd /faiss/build/faiss/python \ - && python setup.py bdist_wheel \ - && mkdir -p /wheels \ - && cp dist/*.whl /wheels/ - -RUN ls -lh /wheels/ - -CMD ["ls", "-lh", "/wheels/"] - -FROM scratch AS faiss-gpu-wheel -COPY --from=faiss-gpu-wheel-builder /wheels /wheels diff --git a/docker/base/Dockerfile.mamba-wheel b/docker/base/Dockerfile.mamba-wheel index 06a0754c74b..8c9e7198d14 100644 --- a/docker/base/Dockerfile.mamba-wheel +++ b/docker/base/Dockerfile.mamba-wheel @@ -9,11 +9,11 @@ # Both only ship source distributions on PyPI and require nvcc to compile. # The two builds are independent stages so BuildKit runs them in parallel. # Each image stores its wheel at /wheels/*.whl. -# Build via the gpu-wheels or nmp-automodel-gpu-wheels bake group. +# Build via the nmp-automodel-gpu-wheels bake group. # # Build args: # CAUSAL_CONV1D_VERSION - git tag to build (default: v1.5.3) -# MAMBA_22_COMMIT - git SHA or tag to build +# MAMBA_22_COMMIT - git SHA or tag to build # CUDA_VERSION - CUDA devel image version (default: 12.8.1) # TORCH_CUDA_ARCH_LIST - semicolon-separated SM targets (default: "8.0;8.6;9.0") ####### @@ -97,7 +97,7 @@ COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /usr/local/bin/uv # ============================================================================= -# causal-conv1d wheel — Python 3.11 (for safe-synthesizer-tasks, nmp-customizer-tasks, nmp-automodel-tasks, and nmp-automodel-tasks) +# causal-conv1d wheel - Python 3.11 # ============================================================================= FROM mamba-wheel-base AS causal-conv1d-wheel-builder @@ -139,7 +139,7 @@ RUN mkdir -p /wheels && \ rm -rf /src/causal-conv1d # The final causal-conv1d-wheel image contains: -# - causal_conv1d-*-cp311-*.whl (for Python 3.11 consumers: safe-synthesizer-tasks, nmp-customizer-tasks, nmp-automodel-tasks) +# - causal_conv1d-*-cp311-*.whl (for Python 3.11 consumers) # - causal_conv1d-*-cp312-*.whl (for Python 3.12 consumers) # Consumers must pin the Python tag glob (e.g. causal_conv1d-*cp311*.whl) to select the right one. FROM scratch AS causal-conv1d-wheel @@ -181,22 +181,6 @@ RUN mkdir -p /wheels && \ uv build --wheel --no-build-isolation --out-dir=/wheels . && \ rm -rf /src/mamba -# ============================================================================= -# mamba-ssm 2.3.0 wheel — Python 3.11 (for nmp-customizer-tasks) -# ============================================================================= -FROM mamba-wheel-base AS mamba-ssm-23-wheel-builder - -ARG MAMBA_23_COMMIT=v2.3.0 - -RUN mkdir -p /wheels && \ - git clone https://github.com/state-spaces/mamba.git /src/mamba && \ - cd /src/mamba && \ - git checkout ${MAMBA_23_COMMIT} && \ - sed -i "/triton/d" setup.py && \ - sed -i "/triton/d" pyproject.toml && \ - uv build --wheel --no-build-isolation --out-dir=/wheels . && \ - rm -rf /src/mamba - # ============================================================================= # mamba-ssm 2.3.0 wheel — Python 3.12 (for Python 3.12 consumers) # ============================================================================= @@ -231,15 +215,14 @@ RUN mkdir -p /wheels && \ rm -rf /src/mamba -# The final mamba-ssm-wheel image contains four versions: -# - mamba_ssm-2.2.5-cp311-*.whl (from MAMBA_22_COMMIT=6b32be06, for safe-synthesizer-tasks / Python 3.11) -# - mamba_ssm-2.2.5-cp312-*.whl (from MAMBA_22_COMMIT=6b32be06, for Python 3.12 consumers, e.g. automodel) -# - mamba_ssm-2.3.0-cp311-*.whl (from v2.3.0, for nmp-customizer-tasks / Python 3.11) +# The final mamba-ssm-wheel image contains: +# - mamba_ssm-2.2.5-cp311-*.whl (from MAMBA_22_COMMIT=6b32be06, for Python 3.11 consumers) +# - mamba_ssm-2.2.5-cp312-*.whl (from MAMBA_22_COMMIT=6b32be06, for Python 3.12 consumers) # - mamba_ssm-2.3.0-cp312-*.whl (from v2.3.0, for Python 3.12 consumers) +# - cu13.1.1/mamba_ssm-2.3.0-cp312-*.whl (from v2.3.0, for Automodel on CUDA 13.1.1) # Consumers must pin both version AND Python tag glob to select the correct wheel. FROM scratch AS mamba-ssm-wheel COPY --from=mamba-ssm-wheel-builder /wheels /wheels COPY --from=mamba-ssm-25-wheel-builder-py312 /wheels /wheels -COPY --from=mamba-ssm-23-wheel-builder /wheels /wheels COPY --from=mamba-ssm-23-wheel-builder-py312 /wheels /wheels COPY --from=mamba-ssm-23-wheel-builder-py312-cu13.1.1 /wheels /wheels/cu13.1.1 diff --git a/docker/base/Dockerfile.nmp-gpu-base-py312 b/docker/base/Dockerfile.nmp-gpu-base-py312 index c6ef88a91ff..ae223278af2 100644 --- a/docker/base/Dockerfile.nmp-gpu-base-py312 +++ b/docker/base/Dockerfile.nmp-gpu-base-py312 @@ -2,7 +2,6 @@ ####### # NMP GPU Base Image (Python 3.12) # Base for GPU task containers that require Python 3.12 -# (e.g. nmp-customizer-tasks) ####### ARG CUDA_VERSION=12.8.1