From 7b30fd2702de290470420dd63c8ea1a92e51b27e Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 12 Feb 2026 15:09:30 -0800 Subject: [PATCH 1/7] fix example script in mtp doc Signed-off-by: Chen Cui --- docs/training/multi-token-prediction.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/training/multi-token-prediction.md b/docs/training/multi-token-prediction.md index d7c3f63b2b..3cfbd5a149 100644 --- a/docs/training/multi-token-prediction.md +++ b/docs/training/multi-token-prediction.md @@ -66,12 +66,13 @@ where: Here's a minimal example using the Qwen3 30B-A3B recipe with MTP enabled: ```python -from megatron.bridge.recipes.qwen import qwen3_30b_a3b_pretrain +from megatron.bridge.recipes.qwen.qwen3_moe import qwen3_30b_a3b_pretrain_config from megatron.bridge.training.pretrain import pretrain +from megatron.bridge.training.gpt_step import forward_step +from megatron.bridge.training.config import ConfigContainer -log_dir = f"/path/to/log/dir" +log_dir = "/path/to/log/dir" cfg: ConfigContainer = qwen3_30b_a3b_pretrain_config() -cfg.logger.log_dir = log_dir cfg.logger.tensorboard_dir = log_dir + "/tb_logs" cfg.checkpoint.save = log_dir + "/checkpoints" cfg.checkpoint.load = log_dir + "/checkpoints" @@ -82,10 +83,11 @@ cfg.dataset.blend=[[ ], None] cfg.dataset.split="9999,8,2" cfg.dataset.path_to_cache = "/path/to/cache" +# cfg.model.num_layers = 8 # train a smaller model if OOM # MTP Configuration -cfg.mtp_num_layers = 1 -cfg.mtp_loss_scaling_factor = 0.1 -pretrain(cfg) +cfg.model.mtp_num_layers = 1 +cfg.model.mtp_loss_scaling_factor = 0.1 +pretrain(cfg, forward_step) ``` Follow the [DCLM Tutorial](https://github.com/NVIDIA-NeMo/Megatron-Bridge/tree/main/tutorials/data/dclm) to prepare the training data From c4abbb9a0fa94c8b68568bf42569a1bb06ddc991 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 12 Feb 2026 15:12:00 -0800 Subject: [PATCH 2/7] add transformers v5 note to ministral scripts Signed-off-by: Chen Cui --- examples/models/vlm/ministral3/conversion.sh | 9 ++++++--- examples/models/vlm/ministral3/inference.sh | 9 ++++++--- examples/models/vlm/ministral3/peft.sh | 5 ++++- examples/models/vlm/ministral3/sft.sh | 5 ++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/examples/models/vlm/ministral3/conversion.sh b/examples/models/vlm/ministral3/conversion.sh index 296af05d3c..965b492658 100755 --- a/examples/models/vlm/ministral3/conversion.sh +++ b/examples/models/vlm/ministral3/conversion.sh @@ -16,17 +16,20 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Note: Ministral 3 requires transformers version 5 +# pip install --upgrade transformers + # Import HF → Megatron -uv run python examples/conversion/convert_checkpoints.py import \ +python examples/conversion/convert_checkpoints.py import \ --hf-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --megatron-path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16 # Export Megatron → HF -uv run python examples/conversion/convert_checkpoints.py export \ +python examples/conversion/convert_checkpoints.py export \ --hf-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --megatron-path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16/iter_0000000 \ --hf-path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16-hf-export # Round-trip validation -uv run python -m torch.distributed.run --nproc_per_node=8 examples/conversion/hf_megatron_roundtrip_multi_gpu.py \ +python -m torch.distributed.run --nproc_per_node=8 examples/conversion/hf_megatron_roundtrip_multi_gpu.py \ --hf-model-id mistralai/Ministral-3-3B-Instruct-2512-BF16 --tp 2 --pp 2 diff --git a/examples/models/vlm/ministral3/inference.sh b/examples/models/vlm/ministral3/inference.sh index 98e20c2050..fad5c78268 100755 --- a/examples/models/vlm/ministral3/inference.sh +++ b/examples/models/vlm/ministral3/inference.sh @@ -16,8 +16,11 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Note: Ministral 3 requires transformers version 5 +# pip install --upgrade transformers + # Inference with Hugging Face checkpoints -uv run python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ +torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --hf_model_path mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --image_path "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/table.png" \ --prompt "Describe this image." \ @@ -26,7 +29,7 @@ uv run python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf --pp 2 # Inference with imported Megatron checkpoints -uv run python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ +torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --hf_model_path mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --megatron_model_path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16/iter_0000000 \ --image_path "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/table.png" \ @@ -36,7 +39,7 @@ uv run python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf --pp 2 # Inference with exported HF checkpoints -uv run python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ +torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --hf_model_path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16-hf-export \ --image_path "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/table.png" \ --prompt "Describe this image." \ diff --git a/examples/models/vlm/ministral3/peft.sh b/examples/models/vlm/ministral3/peft.sh index 0fb8e1b38e..5c4bcc5f0c 100755 --- a/examples/models/vlm/ministral3/peft.sh +++ b/examples/models/vlm/ministral3/peft.sh @@ -16,6 +16,9 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Note: Ministral 3 requires transformers version 5 +# pip install --upgrade transformers + # Common configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16 MODEL_NAME=ministral3_3b @@ -38,7 +41,7 @@ for config in "${PARALLELISM_CONFIGS[@]}"; do IFS=',' read -r TP PP <<< "$config" echo "Running LoRA finetuning with TP=$TP, PP=$PP" - uv run python -m torch.distributed.run --nproc_per_node=8 scripts/training/run_recipe.py \ + torchrun --nproc_per_node=8 scripts/training/run_recipe.py \ --recipe ${MODEL_NAME}_finetune_config \ --step_func vlm_step \ --peft_scheme lora \ diff --git a/examples/models/vlm/ministral3/sft.sh b/examples/models/vlm/ministral3/sft.sh index 193afaf10e..40aa0e67da 100755 --- a/examples/models/vlm/ministral3/sft.sh +++ b/examples/models/vlm/ministral3/sft.sh @@ -16,6 +16,9 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Note: Ministral 3 requires transformers version 5 +# pip install --upgrade transformers + # Common configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16 MODEL_NAME=ministral3_3b @@ -38,7 +41,7 @@ for config in "${PARALLELISM_CONFIGS[@]}"; do IFS=',' read -r TP PP <<< "$config" echo "Running full finetuning with TP=$TP, PP=$PP" - uv run python -m torch.distributed.run --nproc_per_node=8 scripts/training/run_recipe.py \ + torchrun --nproc_per_node=8 scripts/training/run_recipe.py \ --recipe ${MODEL_NAME}_finetune_config \ --step_func vlm_step \ checkpoint.pretrained_checkpoint=$PRETRAINED_CHECKPOINT \ From 8e1c9bce32a53c6d9bd561ffacd395ec268e95a3 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 12 Feb 2026 15:15:41 -0800 Subject: [PATCH 3/7] add note on wandb in training scripts Signed-off-by: Chen Cui --- examples/models/vlm/gemma3_vl/peft.sh | 4 ++++ examples/models/vlm/gemma3_vl/sft.sh | 4 ++++ examples/models/vlm/glm_45v/slurm_peft.sh | 4 ++++ examples/models/vlm/glm_45v/slurm_sft.sh | 4 ++++ examples/models/vlm/ministral3/peft.sh | 4 ++++ examples/models/vlm/ministral3/sft.sh | 4 ++++ examples/models/vlm/qwen3_vl/peft.sh | 4 ++++ examples/models/vlm/qwen3_vl/sft.sh | 4 ++++ 8 files changed, 32 insertions(+) diff --git a/examples/models/vlm/gemma3_vl/peft.sh b/examples/models/vlm/gemma3_vl/peft.sh index a4786e14d3..c966900ee4 100644 --- a/examples/models/vlm/gemma3_vl/peft.sh +++ b/examples/models/vlm/gemma3_vl/peft.sh @@ -16,6 +16,10 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Common configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/gemma-3-4b-it MODEL_NAME=gemma3_vl_4b diff --git a/examples/models/vlm/gemma3_vl/sft.sh b/examples/models/vlm/gemma3_vl/sft.sh index b7715c4eaf..820c9c3298 100755 --- a/examples/models/vlm/gemma3_vl/sft.sh +++ b/examples/models/vlm/gemma3_vl/sft.sh @@ -16,6 +16,10 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Common configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/gemma-3-4b-it MODEL_NAME=gemma3_vl_4b diff --git a/examples/models/vlm/glm_45v/slurm_peft.sh b/examples/models/vlm/glm_45v/slurm_peft.sh index 017da6a74c..066bbf15fa 100755 --- a/examples/models/vlm/glm_45v/slurm_peft.sh +++ b/examples/models/vlm/glm_45v/slurm_peft.sh @@ -44,6 +44,10 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Model and training configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/GLM-4.5V MODEL_NAME=glm_45v diff --git a/examples/models/vlm/glm_45v/slurm_sft.sh b/examples/models/vlm/glm_45v/slurm_sft.sh index f23dee3c43..3754de0a0b 100644 --- a/examples/models/vlm/glm_45v/slurm_sft.sh +++ b/examples/models/vlm/glm_45v/slurm_sft.sh @@ -44,6 +44,10 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Model and training configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/GLM-4.5V MODEL_NAME=glm_45v diff --git a/examples/models/vlm/ministral3/peft.sh b/examples/models/vlm/ministral3/peft.sh index 5c4bcc5f0c..791f1e7273 100755 --- a/examples/models/vlm/ministral3/peft.sh +++ b/examples/models/vlm/ministral3/peft.sh @@ -19,6 +19,10 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 # pip install --upgrade transformers +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Common configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16 MODEL_NAME=ministral3_3b diff --git a/examples/models/vlm/ministral3/sft.sh b/examples/models/vlm/ministral3/sft.sh index 40aa0e67da..81d0891f92 100755 --- a/examples/models/vlm/ministral3/sft.sh +++ b/examples/models/vlm/ministral3/sft.sh @@ -19,6 +19,10 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 # pip install --upgrade transformers +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Common configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16 MODEL_NAME=ministral3_3b diff --git a/examples/models/vlm/qwen3_vl/peft.sh b/examples/models/vlm/qwen3_vl/peft.sh index 788be80939..6cddb470a0 100644 --- a/examples/models/vlm/qwen3_vl/peft.sh +++ b/examples/models/vlm/qwen3_vl/peft.sh @@ -16,6 +16,10 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Common configurations for dense model finetuning PRETRAINED_CHECKPOINT=${WORKSPACE}/models/Qwen3-VL-8B-Instruct MODEL_NAME=qwen3_vl_8b diff --git a/examples/models/vlm/qwen3_vl/sft.sh b/examples/models/vlm/qwen3_vl/sft.sh index 7732b38428..0a26786273 100755 --- a/examples/models/vlm/qwen3_vl/sft.sh +++ b/examples/models/vlm/qwen3_vl/sft.sh @@ -16,6 +16,10 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} +# Before training, make sure to set WANDB_API_KEY or disable wandb logging +# export WANDB_API_KEY= +# export WANDB_MODE=disabled + # Common configurations for dense model finetuning PRETRAINED_CHECKPOINT=${WORKSPACE}/models/Qwen3-VL-8B-Instruct MODEL_NAME=qwen3_vl_8b From 604f92808b049b3be1babef98d5f9ef5ebda0043 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 12 Feb 2026 15:25:45 -0800 Subject: [PATCH 4/7] add note why uv run is not used Signed-off-by: Chen Cui --- examples/models/vlm/ministral3/conversion.sh | 1 + examples/models/vlm/ministral3/inference.sh | 1 + examples/models/vlm/ministral3/peft.sh | 1 + examples/models/vlm/ministral3/sft.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/examples/models/vlm/ministral3/conversion.sh b/examples/models/vlm/ministral3/conversion.sh index 965b492658..7281efa980 100755 --- a/examples/models/vlm/ministral3/conversion.sh +++ b/examples/models/vlm/ministral3/conversion.sh @@ -18,6 +18,7 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 # pip install --upgrade transformers +# Commands below use python instead of uv run to avoid conflicts with the virtual environment. # Import HF → Megatron python examples/conversion/convert_checkpoints.py import \ diff --git a/examples/models/vlm/ministral3/inference.sh b/examples/models/vlm/ministral3/inference.sh index fad5c78268..fc95afe865 100755 --- a/examples/models/vlm/ministral3/inference.sh +++ b/examples/models/vlm/ministral3/inference.sh @@ -18,6 +18,7 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 # pip install --upgrade transformers +# Commands below use torchrun instead of uv run to avoid conflicts with the virtual environment. # Inference with Hugging Face checkpoints torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ diff --git a/examples/models/vlm/ministral3/peft.sh b/examples/models/vlm/ministral3/peft.sh index 791f1e7273..182670c9f8 100755 --- a/examples/models/vlm/ministral3/peft.sh +++ b/examples/models/vlm/ministral3/peft.sh @@ -18,6 +18,7 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 # pip install --upgrade transformers +# Commands below use torchrun instead of uv run to avoid conflicts with the virtual environment. # Before training, make sure to set WANDB_API_KEY or disable wandb logging # export WANDB_API_KEY= diff --git a/examples/models/vlm/ministral3/sft.sh b/examples/models/vlm/ministral3/sft.sh index 81d0891f92..bb948e06b8 100755 --- a/examples/models/vlm/ministral3/sft.sh +++ b/examples/models/vlm/ministral3/sft.sh @@ -18,6 +18,7 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 # pip install --upgrade transformers +# Commands below use torchrun instead of uv run to avoid conflicts with the virtual environment. # Before training, make sure to set WANDB_API_KEY or disable wandb logging # export WANDB_API_KEY= From 7c60891c66119d13b7193bc7d7e75e7dfeea2058 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 12 Feb 2026 15:27:30 -0800 Subject: [PATCH 5/7] remove duplicate wandb Signed-off-by: Chen Cui --- examples/models/vlm/glm_45v/slurm_peft.sh | 8 +++----- examples/models/vlm/glm_45v/slurm_sft.sh | 6 ++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/models/vlm/glm_45v/slurm_peft.sh b/examples/models/vlm/glm_45v/slurm_peft.sh index 066bbf15fa..692f5bffb7 100755 --- a/examples/models/vlm/glm_45v/slurm_peft.sh +++ b/examples/models/vlm/glm_45v/slurm_peft.sh @@ -44,15 +44,11 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} -# Before training, make sure to set WANDB_API_KEY or disable wandb logging -# export WANDB_API_KEY= -# export WANDB_MODE=disabled - # Model and training configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/GLM-4.5V MODEL_NAME=glm_45v DATASET_NAME=cord_v2 -SEQ_LENGTH=8192 +SEQ_LENGTH=8192∂ TRAIN_ITERS=50 GLOBAL_BATCH_SIZE=32 MICRO_BATCH_SIZE=1 @@ -94,6 +90,8 @@ export NCCL_NVLS_ENABLE=0 # Authentication tokens (set these for your environment) # export HF_TOKEN="hf_your_token_here" # export WANDB_API_KEY="your_wandb_key_here" +# or disable wandb logging +# export WANDB_MODE=disabled # ============================================================================== # Job Execution diff --git a/examples/models/vlm/glm_45v/slurm_sft.sh b/examples/models/vlm/glm_45v/slurm_sft.sh index 3754de0a0b..e76e6968a1 100644 --- a/examples/models/vlm/glm_45v/slurm_sft.sh +++ b/examples/models/vlm/glm_45v/slurm_sft.sh @@ -44,10 +44,6 @@ # Workspace directory for checkpoints and results WORKSPACE=${WORKSPACE:-/workspace} -# Before training, make sure to set WANDB_API_KEY or disable wandb logging -# export WANDB_API_KEY= -# export WANDB_MODE=disabled - # Model and training configurations PRETRAINED_CHECKPOINT=${WORKSPACE}/models/GLM-4.5V MODEL_NAME=glm_45v @@ -94,6 +90,8 @@ export NCCL_NVLS_ENABLE=0 # Authentication tokens (set these for your environment) # export HF_TOKEN="hf_your_token_here" # export WANDB_API_KEY="your_wandb_key_here" +# or disable wandb logging +# export WANDB_MODE=disabled # ============================================================================== # Job Execution From 6695869ff304e55f5d1b8c743faaf701893a869f Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 12 Feb 2026 15:28:20 -0800 Subject: [PATCH 6/7] typo Signed-off-by: Chen Cui --- examples/models/vlm/glm_45v/slurm_peft.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/models/vlm/glm_45v/slurm_peft.sh b/examples/models/vlm/glm_45v/slurm_peft.sh index 692f5bffb7..e876089c34 100755 --- a/examples/models/vlm/glm_45v/slurm_peft.sh +++ b/examples/models/vlm/glm_45v/slurm_peft.sh @@ -48,7 +48,7 @@ WORKSPACE=${WORKSPACE:-/workspace} PRETRAINED_CHECKPOINT=${WORKSPACE}/models/GLM-4.5V MODEL_NAME=glm_45v DATASET_NAME=cord_v2 -SEQ_LENGTH=8192∂ +SEQ_LENGTH=8192 TRAIN_ITERS=50 GLOBAL_BATCH_SIZE=32 MICRO_BATCH_SIZE=1 From 93d6f3576e16ad3e56b6363c736d096c3b0ff204 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Thu, 12 Feb 2026 15:51:03 -0800 Subject: [PATCH 7/7] use uv run no sync Signed-off-by: Chen Cui --- examples/models/vlm/ministral3/conversion.sh | 10 +++++----- examples/models/vlm/ministral3/inference.sh | 10 +++++----- examples/models/vlm/ministral3/peft.sh | 6 +++--- examples/models/vlm/ministral3/sft.sh | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/models/vlm/ministral3/conversion.sh b/examples/models/vlm/ministral3/conversion.sh index 7281efa980..7b0bbad008 100755 --- a/examples/models/vlm/ministral3/conversion.sh +++ b/examples/models/vlm/ministral3/conversion.sh @@ -17,20 +17,20 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 -# pip install --upgrade transformers -# Commands below use python instead of uv run to avoid conflicts with the virtual environment. +# uv pip install --upgrade transformers +# Commands below use uv run --no-sync to avoid conflicts with the virtual environment. # Import HF → Megatron -python examples/conversion/convert_checkpoints.py import \ +uv run --no-sync python examples/conversion/convert_checkpoints.py import \ --hf-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --megatron-path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16 # Export Megatron → HF -python examples/conversion/convert_checkpoints.py export \ +uv run --no-sync python examples/conversion/convert_checkpoints.py export \ --hf-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --megatron-path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16/iter_0000000 \ --hf-path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16-hf-export # Round-trip validation -python -m torch.distributed.run --nproc_per_node=8 examples/conversion/hf_megatron_roundtrip_multi_gpu.py \ +uv run --no-sync python -m torch.distributed.run --nproc_per_node=8 examples/conversion/hf_megatron_roundtrip_multi_gpu.py \ --hf-model-id mistralai/Ministral-3-3B-Instruct-2512-BF16 --tp 2 --pp 2 diff --git a/examples/models/vlm/ministral3/inference.sh b/examples/models/vlm/ministral3/inference.sh index fc95afe865..de0b8bee29 100755 --- a/examples/models/vlm/ministral3/inference.sh +++ b/examples/models/vlm/ministral3/inference.sh @@ -17,11 +17,11 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 -# pip install --upgrade transformers -# Commands below use torchrun instead of uv run to avoid conflicts with the virtual environment. +# uv pip install --upgrade transformers +# Commands below use uv run --no-sync to avoid conflicts with the virtual environment. # Inference with Hugging Face checkpoints -torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ +uv run --no-sync python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --hf_model_path mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --image_path "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/table.png" \ --prompt "Describe this image." \ @@ -30,7 +30,7 @@ torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --pp 2 # Inference with imported Megatron checkpoints -torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ +uv run --no-sync python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --hf_model_path mistralai/Ministral-3-3B-Instruct-2512-BF16 \ --megatron_model_path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16/iter_0000000 \ --image_path "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/table.png" \ @@ -40,7 +40,7 @@ torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --pp 2 # Inference with exported HF checkpoints -torchrun --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ +uv run --no-sync python -m torch.distributed.run --nproc_per_node=4 examples/conversion/hf_to_megatron_generate_vlm.py \ --hf_model_path ${WORKSPACE}/models/Ministral-3-3B-Instruct-2512-BF16-hf-export \ --image_path "https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16/resolve/main/images/table.png" \ --prompt "Describe this image." \ diff --git a/examples/models/vlm/ministral3/peft.sh b/examples/models/vlm/ministral3/peft.sh index 182670c9f8..b3c44a2f86 100755 --- a/examples/models/vlm/ministral3/peft.sh +++ b/examples/models/vlm/ministral3/peft.sh @@ -17,8 +17,8 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 -# pip install --upgrade transformers -# Commands below use torchrun instead of uv run to avoid conflicts with the virtual environment. +# uv pip install --upgrade transformers +# Commands below use uv run --no-sync to avoid conflicts with the virtual environment. # Before training, make sure to set WANDB_API_KEY or disable wandb logging # export WANDB_API_KEY= @@ -46,7 +46,7 @@ for config in "${PARALLELISM_CONFIGS[@]}"; do IFS=',' read -r TP PP <<< "$config" echo "Running LoRA finetuning with TP=$TP, PP=$PP" - torchrun --nproc_per_node=8 scripts/training/run_recipe.py \ + uv run --no-sync python -m torch.distributed.run --nproc_per_node=8 scripts/training/run_recipe.py \ --recipe ${MODEL_NAME}_finetune_config \ --step_func vlm_step \ --peft_scheme lora \ diff --git a/examples/models/vlm/ministral3/sft.sh b/examples/models/vlm/ministral3/sft.sh index bb948e06b8..a22eebbb03 100755 --- a/examples/models/vlm/ministral3/sft.sh +++ b/examples/models/vlm/ministral3/sft.sh @@ -17,8 +17,8 @@ WORKSPACE=${WORKSPACE:-/workspace} # Note: Ministral 3 requires transformers version 5 -# pip install --upgrade transformers -# Commands below use torchrun instead of uv run to avoid conflicts with the virtual environment. +# uv pip install --upgrade transformers +# Commands below use uv run --no-sync to avoid conflicts with the virtual environment. # Before training, make sure to set WANDB_API_KEY or disable wandb logging # export WANDB_API_KEY= @@ -46,7 +46,7 @@ for config in "${PARALLELISM_CONFIGS[@]}"; do IFS=',' read -r TP PP <<< "$config" echo "Running full finetuning with TP=$TP, PP=$PP" - torchrun --nproc_per_node=8 scripts/training/run_recipe.py \ + uv run --no-sync python -m torch.distributed.run --nproc_per_node=8 scripts/training/run_recipe.py \ --recipe ${MODEL_NAME}_finetune_config \ --step_func vlm_step \ checkpoint.pretrained_checkpoint=$PRETRAINED_CHECKPOINT \