From d1a503b08abd1abe62b3e849982d7d60e0043bf0 Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Wed, 5 Aug 2026 20:25:17 +0000 Subject: [PATCH 1/4] fix(packaging): include megatron.post_training in package discovery Add megatron.post_training and megatron.post_training.* to pyproject.toml's [tool.setuptools.packages.find] include list. Move modelopt imports to lazy (inside functions) so the module is importable without modelopt installed. Same fix as PR #6290 for the dev branch. Signed-off-by: svcnemo-autobot --- megatron/post_training/utils.py | 7 ++++--- pyproject.toml | 9 ++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/megatron/post_training/utils.py b/megatron/post_training/utils.py index 5008e1b33dc..f68836e2d46 100644 --- a/megatron/post_training/utils.py +++ b/megatron/post_training/utils.py @@ -3,10 +3,7 @@ import inspect import os -import modelopt -import modelopt.torch.quantization as mtq import torch -from modelopt.torch.quantization.utils import is_quantized from packaging.version import Version from megatron.core import parallel_state @@ -15,6 +12,7 @@ def modelopt_version_higher_than(target_version: str): """Check if Model-Optimizer is greater than this version.""" + import modelopt info = "rank {:3}/{:3} checking if nvidia-modelopt-{} is higher than {}".format( torch.distributed.get_rank(), torch.distributed.get_world_size(), @@ -27,6 +25,7 @@ def modelopt_version_higher_than(target_version: str): def modelopt_version_at_least(target_version: str): """Check if Model-Optimizer is greater or equal than this version.""" + import modelopt info = "rank {:3}/{:3} checking if nvidia-modelopt-{} is at least {}".format( torch.distributed.get_rank(), torch.distributed.get_world_size(), @@ -101,6 +100,8 @@ def _empty_like_if_meta(tensor: torch.Tensor, *, device: torch.device): def print_distributed_quant_summary(model, msg=""): + import modelopt.torch.quantization as mtq + from modelopt.torch.quantization.utils import is_quantized from megatron.core import parallel_state from megatron.core.utils import unwrap_model from megatron.training import print_rank_0 diff --git a/pyproject.toml b/pyproject.toml index 5405a2edab5..bb6dc21935d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,14 @@ build-backend = "setuptools.build_meta" include-package-data = true [tool.setuptools.packages.find] -include = ["megatron.core", "megatron.core.*", "megatron.training", "megatron.training.*"] +include = [ + "megatron.core", + "megatron.core.*", + "megatron.training", + "megatron.training.*", + "megatron.post_training", + "megatron.post_training.*", +] [tool.setuptools.dynamic] version = { attr = "megatron.core.package_info.__version__" } From 21e808201eef603a5cc01a7649934fd4b5f8de95 Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Thu, 6 Aug 2026 00:53:10 +0000 Subject: [PATCH 2/4] fix(security): pin recipe clones to immutable commits Pin fast-hadamard-transform to f134af63 and FlashMLA to b7643bd in the DSV4 GB200 recipe YAML. Unpinned clones of mutable branch HEADs can execute arbitrary future remote code on CI runners. Signed-off-by: svcnemo-autobot --- .../deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/moe_recipes/deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml b/examples/moe_recipes/deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml index 9fe34ff9ca3..6b3ba805108 100644 --- a/examples/moe_recipes/deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml +++ b/examples/moe_recipes/deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml @@ -52,6 +52,7 @@ DEPENDENCIES: WORKDIR /home/ RUN git clone https://github.com/Dao-AILab/fast-hadamard-transform.git && \ cd fast-hadamard-transform && \ + git checkout f134af63deb2df17e1171a9ec1ea4a7d8604d5ca && \ pip install --no-build-isolation . # Emerging-Optimizers (Muon) @@ -64,6 +65,7 @@ DEPENDENCIES: WORKDIR /opt/ RUN git clone --branch nv_dev https://github.com/deepseek-ai/FlashMLA.git && \ cd FlashMLA && \ + git checkout b7643bd && \ FLASH_MLA_DISABLE_SM90=1 \ NVCC_THREADS=16 \ CFLAGS="-I/usr/local/cuda/include/cccl" \ From f0d50f98bb66aba282432739e3b09ccd0d08db55 Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Thu, 6 Aug 2026 08:33:02 +0000 Subject: [PATCH 3/4] fix(deps): align dev Transformer Engine revision Signed-off-by: svcnemo-autobot --- pyproject.toml | 2 +- uv.lock | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bb6dc21935d..33a08b92d1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,7 +205,7 @@ flash_mla = [ { git = "https://github.com/deepseek-ai/FlashMLA", rev = "nv_dev" }, ] nvidia-cudnn-frontend = { git = "https://github.com/NVIDIA/cudnn-frontend.git", rev = "0a14b7181d129d30e7bad34b8c3ed0a0c995e23d" } -transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "f031cf87bd054c7558b887df7bed93975456667f" } +transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "cfda38340769c432c12db3c0df574f8e6ce46c8c" } nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "17ae86b64d7f75653351664f5d8c9e466faede00" } emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "v0.3.0" } fast-hadamard-transform = { git = "https://github.com/Dao-AILab/fast-hadamard-transform.git", rev = "f134af63deb2df17e1171a9ec1ea4a7d8604d5ca" } diff --git a/uv.lock b/uv.lock index 55dc1bd308b..2fd9a01aefe 100644 --- a/uv.lock +++ b/uv.lock @@ -3091,7 +3091,7 @@ requires-dist = [ { name = "torch", specifier = ">=2.6.0" }, { name = "tqdm", marker = "extra == 'dev'" }, { name = "tqdm", marker = "extra == 'lts'" }, - { name = "transformer-engine", extras = ["core-cu13", "pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=f031cf87bd054c7558b887df7bed93975456667f" }, + { name = "transformer-engine", extras = ["core-cu13", "pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=cfda38340769c432c12db3c0df574f8e6ce46c8c" }, { name = "transformers", marker = "extra == 'mlm'" }, { name = "transformers", marker = "extra == 'training'" }, { name = "wandb", marker = "extra == 'mlm'" }, @@ -7800,12 +7800,13 @@ wheels = [ [[package]] name = "transformer-engine" -version = "2.14.0+f031cf87" -source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=f031cf87bd054c7558b887df7bed93975456667f#f031cf87bd054c7558b887df7bed93975456667f" } +version = "2.18.0+cfda3834" +source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=cfda38340769c432c12db3c0df574f8e6ce46c8c#cfda38340769c432c12db3c0df574f8e6ce46c8c" } dependencies = [ { name = "einops" }, { name = "importlib-metadata" }, { name = "nvdlfw-inspect" }, + { name = "nvidia-cudnn-frontend", version = "1.25.0", source = { git = "https://github.com/NVIDIA/cudnn-frontend.git?rev=0a14b7181d129d30e7bad34b8c3ed0a0c995e23d#0a14b7181d129d30e7bad34b8c3ed0a0c995e23d" } }, { name = "onnx" }, { name = "onnxscript" }, { name = "packaging" }, From 6075846c0f4c44c85b4f8b7480f0b8f88a1315a8 Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Thu, 6 Aug 2026 08:51:47 +0000 Subject: [PATCH 4/4] fix(deps): admit selected Transformer Engine version Signed-off-by: svcnemo-autobot --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 33a08b92d1b..0d8ed7dacf6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,7 +91,7 @@ mlm = [ dev = [ "nvidia-modelopt[torch]; sys_platform != 'darwin'", - "transformer-engine[pytorch,core_cu13]>=2.9.0a0,<2.12.0", + "transformer-engine[pytorch,core_cu13]>=2.18.0,<2.19.0", "nvidia-resiliency-ext==0.6.0", "tqdm", "einops~=0.8",