Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docker-bake.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,12 @@ variable "BASE_TAG_PYTHON" {

# Pin for nmp-automodel-base.
variable "BASE_TAG_AUTOMODEL" {
default = "d9e1851f309d3cf5389c0fc0e1049bd3c87593f8"
default = "65632527b258367fb8855ef431f596abb8538577"
}

# The tag for base images if needed
variable "WHEELS_TAG" {
default = "c6fe97bde6a687d506a33edfee36569b8c7252f3"
default = "65632527b258367fb8855ef431f596abb8538577"
}

variable "BAKE_CACHE_SOURCE_BRANCH" {
Expand Down
40 changes: 28 additions & 12 deletions docker/Dockerfile.nmp-automodel-base
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
# PyTorch NGC base + Automodel + CUDA extension wheels for nmp-automodel images.
# Publish target: nmp-automodel-base (slim image; builder is build-only).

ARG AUTOMODEL_COMMIT=0e9909f56ba48ef9761fc6f49323ba9d0a0835b2
# Pin to the tip of the Automodel release branch (origin/r0.5.0), not main.
# Re-pin AUTOMODEL_COMMIT when cutting to a new r0.x.y.
ARG AUTOMODEL_COMMIT=84e85792e34a240f09194a919c1e62c9d3e47938
ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.05-py3

FROM causal-conv1d-wheel-image AS causal-conv1d-wheel-src
FROM mamba-ssm-wheel-image AS mamba-ssm-wheel-src
Expand All @@ -16,7 +19,8 @@ RUN git clone --branch main https://github.com/NVIDIA-NeMo/Automodel.git /opt/Au
git checkout ${AUTOMODEL_COMMIT} && \
rm -rf /opt/Automodel/.git

FROM nvcr.io/nvidia/pytorch:26.02-py3 AS nmp-automodel-base-builder
FROM ${PYTORCH_BASE} AS nmp-automodel-base-builder
ARG PYTORCH_BASE

WORKDIR /opt

Expand All @@ -32,10 +36,10 @@ RUN uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages

COPY --from=automodel-clone /opt/Automodel /opt/Automodel
COPY docker/automodel/cherry-picks /opt/cherry-picks
RUN cd /opt/Automodel && patch -p1 < /opt/cherry-picks/e6d2930a.diff

RUN cd /opt/Automodel && \
bash docker/common/update_pyproject_pytorch.sh /opt/Automodel
bash docker/common/update_pyproject_pytorch.sh /opt/Automodel && \
patch -p1 < /opt/cherry-picks/3d98f6e3.diff

# Sync all Automodel extras except cuda — causal-conv1d, mamba-ssm, nv-grouped-gemm,
# and bitsandbytes are installed from prebuilt wheels / source in the steps below.
Expand All @@ -60,16 +64,16 @@ RUN --mount=type=cache,target=/root/.cache/uv \
RUN --mount=from=causal-conv1d-wheel-src,target=/tmp/causal-conv1d-wheel-src,readonly \
--mount=from=mamba-ssm-wheel-src,target=/tmp/mamba-ssm-wheel-src,readonly \
uv pip install --no-cache-dir --no-deps \
/tmp/causal-conv1d-wheel-src/wheels/cu13.1.1/causal_conv1d-*cp312*.whl \
/tmp/mamba-ssm-wheel-src/wheels/cu13.1.1/mamba_ssm-2.3.0-cp312*.whl
/tmp/causal-conv1d-wheel-src/wheels/cu13.2/causal_conv1d-*cp312*.whl \
/tmp/mamba-ssm-wheel-src/wheels/cu13.2/mamba_ssm-2.3.0-cp312*.whl

RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install --no-build-isolation --no-deps git+https://github.com/fanshiqing/grouped_gemm@v1.1.4

RUN --mount=type=cache,target=/root/.cache/uv \
git clone https://github.com/bitsandbytes-foundation/bitsandbytes.git && \
cd bitsandbytes && \
git checkout 0.49.1 && \
git checkout 0.49.2 && \
cmake -DCOMPUTE_CAPABILITY="75;80;86;87;89;90;100;103;110;120;121" -DCOMPUTE_BACKEND=cuda -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -S . && \
make -j"$(nproc)" && \
uv pip install scikit-build-core --no-deps && \
Expand All @@ -87,20 +91,32 @@ RUN --mount=type=cache,target=/root/.cache/uv \
rm -rf /opt/venv/lib/python*/site-packages/vllm \
/opt/venv/lib/python*/site-packages/vllm-*.dist-info && \
uv pip install \
"black>=26.3.1" \
"pyasn1>=0.6.3" \
"onnx>=1.21.0"
"aiohttp>=3.13.3,<4" \
"black>=26.3.1,<27" \
"GitPython>=3.1.50,<4" \
"jaraco-context>=6.1.0,<7" \
"jupyter-server>=2.18.0,<3" \
"jupyterlab>=4.5.7,<5" \
"mistune>=3.2.1,<4" \
"notebook>=7.5.6,<8" \
"onnx>=1.21.0,<2" \
"pillow>=12.2.0,<13" \
"pyasn1>=0.6.3,<1" \
"tornado>=6.5.5,<7" \
"urllib3>=2.7.0,<3"

# Published base image (same filesystem as builder).
FROM nvcr.io/nvidia/pytorch:26.02-py3 AS nmp-automodel-base
FROM ${PYTORCH_BASE} AS nmp-automodel-base
ARG PYTORCH_BASE
COPY --from=nmp-automodel-base-builder /opt/venv /opt/venv
COPY --from=nmp-automodel-base-builder /opt/Automodel /opt/Automodel
# Builder pins uv 0.9.14 but does not ship it in the venv layer; PyTorch base may ship 0.10.x.
COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv

ENV VIRTUAL_ENV=/opt/venv \
UV_PROJECT_ENVIRONMENT=/opt/venv \
HF_HUB_ENABLE_HF_TRANSFER=1
HF_HUB_ENABLE_HF_TRANSFER=1 \
FORCE_QWENVL_VIDEO_READER=torchcodec
ENV PATH="/bin:/opt/venv/bin:/root/.local/bin:$PATH"
WORKDIR /opt

Expand Down
10 changes: 8 additions & 2 deletions docker/automodel/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Three images for the **nmp-automodel** customization backend. Published as flat

| Image | Dockerfile | Role |
|-------|------------|------|
| `nmp-automodel-base` | `Dockerfile.nmp-automodel-base` | PyTorch 26.02 + Automodel + `mamba-ssm` / `causal-conv1d` wheels |
| `nmp-automodel-base` | `Dockerfile.nmp-automodel-base` | PyTorch 26.05 + Automodel + `mamba-ssm` / `causal-conv1d` wheels |
| `nmp-automodel-tasks` | `Dockerfile.nmp-automodel-tasks` | Platform task glue (`file_io`, `model_entity`, `model_spec`); GPU-capable base |
| `nmp-automodel-training` | `Dockerfile.nmp-automodel-training` | Training step (`nmp.automodel.tasks.training`) |

Expand Down Expand Up @@ -77,7 +77,13 @@ Override registry: `export WHEELS_REGISTRY=...` and `export IMAGE_REGISTRY=...`

## Tasks / training runtime (platform glue)

**Base (`nmp-automodel-base`):** NGC PyTorch 26.02, Automodel `uv sync --locked`, pinned `transformers`/`torch`.
**Base (`nmp-automodel-base`):** NGC PyTorch 26.05, Automodel `uv sync --locked`, pinned `transformers`/`torch`.

**Automodel cherry-picks:** Platform-specific patches under `docker/automodel/cherry-picks/` are applied after `update_pyproject_pytorch.sh` and before `uv sync`. Re-pin or drop patches when upstream `r0.x.y` absorbs the same changes.

| Patch | Purpose |
|-------|---------|
| `3d98f6e3.diff` | Drop `decord` + `imageio-ffmpeg` (old bundled ffmpeg); use `torchcodec` for VLM video (`FORCE_QWENVL_VIDEO_READER=torchcodec`) |

**Tasks image:** `uv sync --package nmp-automodel --no-dev --inexact` from the minimal workspace. CPU steps only need platform SDK glue; upgrading ancillary packages here does not affect training.

Expand Down
119 changes: 119 additions & 0 deletions docker/automodel/cherry-picks/3d98f6e3.diff
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
diff --git a/pyproject.toml b/pyproject.toml
index 801ca1be..3d98f6e3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -99,7 +99,6 @@ diffusion = [
"diffusers>=0.37.0",
"ftfy",
"imageio",
- "imageio-ffmpeg",
"opencv-python-headless",
"torchvision",
]
@@ -154,7 +153,6 @@ vlm = [
"pillow",
"qwen-omni-utils",
"qwen-vl-utils",
- "decord; (platform_machine == 'x86_64' and platform_system != 'Darwin')",
"soundfile",
"timm<=1.0.22",
"torchcodec; (platform_machine == 'x86_64' and platform_system != 'Darwin')",
diff --git a/uv.lock b/uv.lock
index ad6029d3..42f67bd1 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1553,19 +1553,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" },
]

-[[package]]
-name = "decord"
-version = "0.6.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform != 'darwin'" },
- { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13' and platform_machine == 'x86_64' and sys_platform != 'darwin'" },
-]
-wheels = [
- { url = "https://files.pythonhosted.org/packages/11/79/936af42edf90a7bd4e41a6cac89c913d4b47fa48a26b042d5129a9242ee3/decord-0.6.0-py3-none-manylinux2010_x86_64.whl", hash = "sha256:51997f20be8958e23b7c4061ba45d0efcd86bffd5fe81c695d0befee0d442976", size = 13602299, upload-time = "2021-06-14T21:30:55.486Z" },
- { url = "https://files.pythonhosted.org/packages/6c/be/e15b5b866da452e62635a7b27513f31cb581fa2ea9cc9b768b535d62a955/decord-0.6.0-py3-none-win_amd64.whl", hash = "sha256:02665d7c4f1193a330205a791bc128f7e108eb6ae5b67144437a02f700943bad", size = 24733380, upload-time = "2021-06-14T21:30:57.766Z" },
-]
-
[[package]]
name = "deep-ep"
version = "1.2.1+7febc6e"
@@ -2587,20 +2574,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/fe/301e0936b79bcab4cacc7548bf2853fc28dced0a578bab1f7ef53c9aa75b/imageio-2.37.2-py3-none-any.whl", hash = "sha256:ad9adfb20335d718c03de457358ed69f141021a333c40a53e57273d8a5bd0b9b", size = 317646, upload-time = "2025-11-04T14:29:37.948Z" },
]

-[[package]]
-name = "imageio-ffmpeg"
-version = "0.6.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/44/bd/c3343c721f2a1b0c9fc71c1aebf1966a3b7f08c2eea8ed5437a2865611d6/imageio_ffmpeg-0.6.0.tar.gz", hash = "sha256:e2556bed8e005564a9f925bb7afa4002d82770d6b08825078b7697ab88ba1755", size = 25210, upload-time = "2025-01-16T21:34:32.747Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/da/58/87ef68ac83f4c7690961bce288fd8e382bc5f1513860fc7f90a9c1c1c6bf/imageio_ffmpeg-0.6.0-py3-none-macosx_10_9_intel.macosx_10_9_x86_64.whl", hash = "sha256:9d2baaf867088508d4a3458e61eeb30e945c4ad8016025545f66c4b5aaef0a61", size = 24932969, upload-time = "2025-01-16T21:34:20.464Z" },
- { url = "https://files.pythonhosted.org/packages/40/5c/f3d8a657d362cc93b81aab8feda487317da5b5d31c0e1fdfd5e986e55d17/imageio_ffmpeg-0.6.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b1ae3173414b5fc5f538a726c4e48ea97edc0d2cdc11f103afee655c463fa742", size = 21113891, upload-time = "2025-01-16T21:34:00.277Z" },
- { url = "https://files.pythonhosted.org/packages/33/e7/1925bfbc563c39c1d2e82501d8372734a5c725e53ac3b31b4c2d081e895b/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1d47bebd83d2c5fc770720d211855f208af8a596c82d17730aa51e815cdee6dc", size = 25632706, upload-time = "2025-01-16T21:33:53.475Z" },
- { url = "https://files.pythonhosted.org/packages/a0/2d/43c8522a2038e9d0e7dbdf3a61195ecc31ca576fb1527a528c877e87d973/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c7e46fcec401dd990405049d2e2f475e2b397779df2519b544b8aab515195282", size = 29498237, upload-time = "2025-01-16T21:34:13.726Z" },
- { url = "https://files.pythonhosted.org/packages/a0/13/59da54728351883c3c1d9fca1710ab8eee82c7beba585df8f25ca925f08f/imageio_ffmpeg-0.6.0-py3-none-win32.whl", hash = "sha256:196faa79366b4a82f95c0f4053191d2013f4714a715780f0ad2a68ff37483cc2", size = 19652251, upload-time = "2025-01-16T21:34:06.812Z" },
- { url = "https://files.pythonhosted.org/packages/2c/c6/fa760e12a2483469e2bf5058c5faff664acf66cadb4df2ad6205b016a73d/imageio_ffmpeg-0.6.0-py3-none-win_amd64.whl", hash = "sha256:02fa47c83703c37df6bfe4896aab339013f62bf02c5ebf2dce6da56af04ffc0a", size = 31246824, upload-time = "2025-01-16T21:34:28.6Z" },
-]
-
[[package]]
name = "imagesize"
version = "1.4.1"
@@ -3900,13 +3873,11 @@ all = [
{ name = "boto3" },
{ name = "causal-conv1d" },
{ name = "databricks-sql-connector" },
- { name = "decord", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" },
{ name = "deltalake" },
{ name = "diffusers" },
{ name = "flash-linear-attention" },
{ name = "ftfy" },
{ name = "imageio" },
- { name = "imageio-ffmpeg" },
{ name = "mamba-ssm" },
{ name = "mistral-common", extra = ["opencv"] },
{ name = "nemo-run" },
@@ -3959,7 +3930,6 @@ diffusion = [
{ name = "diffusers" },
{ name = "ftfy" },
{ name = "imageio" },
- { name = "imageio-ffmpeg" },
{ name = "opencv-python-headless" },
{ name = "torchvision", version = "0.25.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" },
{ name = "torchvision", version = "0.25.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux'" },
@@ -3969,7 +3939,6 @@ diffusion-kernels = [
{ name = "diffusers" },
{ name = "ftfy" },
{ name = "imageio" },
- { name = "imageio-ffmpeg" },
{ name = "kernels" },
{ name = "opencv-python-headless" },
{ name = "torchvision", version = "0.25.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" },
@@ -4007,7 +3976,6 @@ s3 = [
vlm = [
{ name = "albumentations" },
{ name = "backoff" },
- { name = "decord", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" },
{ name = "mistral-common", extra = ["opencv"] },
{ name = "numba", version = "0.53.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" },
{ name = "numba", version = "0.62.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" },
@@ -4072,7 +4040,6 @@ requires-dist = [
{ name = "causal-conv1d", marker = "extra == 'cuda'" },
{ name = "databricks-sql-connector", marker = "extra == 'delta-databricks'", specifier = ">=3.0.0" },
{ name = "datasets", specifier = ">=4.0.0" },
- { name = "decord", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin' and extra == 'vlm'" },
{ name = "deep-ep", marker = "extra == 'moe'", git = "https://github.com/deepseek-ai/DeepEP.git?rev=7febc6e25660af0f54d95dd781ecdcd62265ecca" },
{ name = "deltalake", marker = "extra == 'delta-databricks'", specifier = ">=1.0.0" },
{ name = "diffusers", marker = "extra == 'diffusion'", specifier = ">=0.37.0" },
@@ -4081,7 +4048,6 @@ requires-dist = [
{ name = "flashoptim", specifier = ">=0.1.3" },
{ name = "ftfy", marker = "extra == 'diffusion'" },
{ name = "imageio", marker = "extra == 'diffusion'" },
- { name = "imageio-ffmpeg", marker = "extra == 'diffusion'" },
{ name = "kernels", marker = "extra == 'diffusion-kernels'" },
{ name = "mamba-ssm", marker = "extra == 'cuda'" },
{ name = "megatron-fsdp", specifier = ">=0.2.3" },
35 changes: 0 additions & 35 deletions docker/automodel/cherry-picks/e6d2930a.diff

This file was deleted.

Loading