diff --git a/docker/Dockerfile.nmp-customizer-tasks b/docker/Dockerfile.nmp-customizer-tasks index 9fc44860f7..084b7ecfe6 100644 --- a/docker/Dockerfile.nmp-customizer-tasks +++ b/docker/Dockerfile.nmp-customizer-tasks @@ -3,11 +3,11 @@ # # Hosts file_io / model_entity (nmp.customization_common.tasks.*), model_spec # analysis (nmp.core.models.tasks.model_spec), and the LoRA adapter sidecar. -# Built on NGC PyTorch 26.05 + transformers/accelerate + Nemotron wheels +# Built on NGC PyTorch 26.07 + transformers/accelerate + Nemotron wheels # (mamba-ssm, causal-conv1d) + platform glue + nmp-models. ARG SMOKE_MARKER=smoke_nmp_customizer_tasks -ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.05-py3 +ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.07-py3 FROM ${PYTORCH_BASE} AS base @@ -48,8 +48,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "transformers==${TRANSFORMERS_VERSION}" \ "accelerate>=1.0.0" -# Nemotron/Mamba CUDA wheels — compiled against NGC 26.05 torch; install --no-deps -# after the HF stack so a PyPI torch never lands in the venv first. +# Nemotron/Mamba CUDA wheels install --no-deps after the HF stack so a PyPI +# torch never lands in the venv first. RUN --mount=from=causal-conv1d-wheel-src,target=/tmp/causal-conv1d-wheel-src,readonly \ --mount=from=mamba-ssm-wheel-src,target=/tmp/mamba-ssm-wheel-src,readonly \ uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache-dir --no-deps \ @@ -68,7 +68,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ -e /app/services/core/models # Pin scanner-tracked runtime packages into /opt/venv before removing stale -# NGC system-site copies inherited from nvcr.io/nvidia/pytorch:26.05-py3. +# NGC system-site copies inherited from the PyTorch base. RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache \ --overrides /app/docker/customizer/preserve_base_torch.txt \ diff --git a/docker/Dockerfile.nmp-unsloth-training b/docker/Dockerfile.nmp-unsloth-training index f432ac6050..467f1551ca 100644 --- a/docker/Dockerfile.nmp-unsloth-training +++ b/docker/Dockerfile.nmp-unsloth-training @@ -28,10 +28,10 @@ # Publish target: nmp-unsloth-training # Default tag: `local` (override via BAKE_TAG at build time). -# NGC PyTorch base. 26.05-py3 ships PyTorch 2.12 + CUDA 13.2 + Python 3.12 (aligned with -# docker/automodel/Dockerfile.nmp-automodel-base). Override at build time: +# NGC PyTorch base, aligned with docker/automodel/Dockerfile.nmp-automodel-base. +# Override at build time: # --set nmp-unsloth-training.args.PYTORCH_BASE=... -ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.05-py3 +ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.07-py3 # Prebuilt CUDA-extension wheels (mamba-ssm + causal-conv1d), shared with # docker/automodel/Dockerfile.nmp-automodel-base. The bake `nmp-unsloth-training` target @@ -124,7 +124,7 @@ RUN --mount=from=causal-conv1d-wheel-src,target=/tmp/causal-conv1d-wheel-src,rea /tmp/causal-conv1d-wheel-src/wheels/cu13.2/causal_conv1d-*cp312*.whl \ /tmp/mamba-ssm-wheel-src/wheels/cu13.2/mamba_ssm-2.3.0-cp312*.whl -# TODO: Step 1d: Flash Attention 2 — compiled from source against the NGC 26.02 torch. +# TODO: Step 1d: Flash Attention 2 — compile from source against the active NGC torch. # /usr/local/cuda symlinks to an older toolkit; use /usr/local/cuda-13.1 instead. # Cap parallel nvcc/ninja work — default uses all CPUs and OOMs typical build hosts. # Put flash attention back in when we have a working wheel in a separate image. @@ -182,7 +182,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "soupsieve>=2.8.4,<3" \ "mlflow-skinny>=3.11.1,<3.12.0" -# Stale NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.05-py3). +# Stale NGC system site-packages inherited from the PyTorch base. RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --only-upgrade openssl libssl3t64 && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/automodel/Dockerfile.nmp-automodel-base b/docker/automodel/Dockerfile.nmp-automodel-base index ecb37c0ba5..5fe174b1a0 100644 --- a/docker/automodel/Dockerfile.nmp-automodel-base +++ b/docker/automodel/Dockerfile.nmp-automodel-base @@ -7,7 +7,7 @@ # Pin to the tip of the Automodel release branch (origin/r0.5.0), not main. # Re-pin AUTOMODEL_COMMIT when cutting to a new r0.x.y. ARG AUTOMODEL_COMMIT=84e85792e34a240f09194a919c1e62c9d3e47938 -ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.05-py3 +ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.07-py3 FROM causal-conv1d-wheel-image AS causal-conv1d-wheel-src FROM mamba-ssm-wheel-image AS mamba-ssm-wheel-src @@ -121,7 +121,7 @@ COPY --from=nmp-automodel-base-builder /opt/Automodel /opt/Automodel # Builder pins uv 0.9.14 but does not ship it in the venv layer; PyTorch base may ship 0.10.x. COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv -# Stale copies under NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.05-py3). +# Stale copies under NGC system site-packages inherited from the PyTorch base. # CVE scanners read dist-packages even when --system-site-packages resolves imports from /opt/venv. # rm -rf (not pip uninstall) avoids corrupted dist-info RECORDs on the NGC base image. RUN apt-get update && \ diff --git a/docker/automodel/README.md b/docker/automodel/README.md index 081f4bec6b..95867425bb 100644 --- a/docker/automodel/README.md +++ b/docker/automodel/README.md @@ -6,7 +6,7 @@ Two images for the **nmp-automodel** customization backend, plus the shared **`n | Image | Dockerfile | Role | |-------|------------|------| -| `nmp-automodel-base` | `docker/automodel/Dockerfile.nmp-automodel-base` | PyTorch 26.05 + Automodel + `mamba-ssm` / `causal-conv1d` wheels | +| `nmp-automodel-base` | `docker/automodel/Dockerfile.nmp-automodel-base` | PyTorch 26.07 + Automodel + `mamba-ssm` / `causal-conv1d` wheels | | `nmp-customizer-tasks` | `docker/Dockerfile.nmp-customizer-tasks` | Shared CPU tasks (`file_io`, `model_entity`, `model_spec`, LoRA sidecar) | | `nmp-automodel-training` | `docker/automodel/Dockerfile.nmp-automodel-training` | Training step (`nmp.automodel.tasks.training`) | @@ -79,7 +79,7 @@ Override registry: `export WHEELS_REGISTRY=...` and `export IMAGE_REGISTRY=...` ## Tasks / training runtime (platform glue) -**Base (`nmp-automodel-base`):** NGC PyTorch 26.05, Automodel `uv sync --locked`, pinned `transformers`/`torch`. +**Base (`nmp-automodel-base`):** NGC PyTorch 26.07, Automodel `uv sync --locked`, pinned `transformers`/`torch`. **Automodel cherry-picks:** Platform-specific patches under `docker/automodel/cherry-picks/` are applied after `update_pyproject_pytorch.sh` and before `uv sync`. Re-pin or drop patches when upstream `r0.x.y` absorbs the same changes. diff --git a/docker/base/Dockerfile.python-wheels b/docker/base/Dockerfile.python-wheels index a38d605c58..8542b0f43d 100644 --- a/docker/base/Dockerfile.python-wheels +++ b/docker/base/Dockerfile.python-wheels @@ -100,8 +100,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends software-proper COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /usr/local/bin/uv -# special builder for 13.2 cuda (automodel on NGC 26.05) -FROM nvcr.io/nvidia/pytorch:26.05-py3 AS mamba-wheel-base-py312-cu13.2 +# special builder for 13.2 cuda (Automodel/Customizer on the current NGC PyTorch base) +FROM nvcr.io/nvidia/pytorch:26.07-py3 AS mamba-wheel-base-py312-cu13.2 ENV DEBIAN_FRONTEND=noninteractive