From 1e50e974cf906e95c6af9b31623b98fd93111ffe Mon Sep 17 00:00:00 2001 From: Dennis Liu Date: Wed, 20 May 2026 17:44:59 +0800 Subject: [PATCH 1/6] docs: add MoE release recipes --- .../b200/mxfp8_256GPU_TP1PP8EP32.yaml | 216 ++++++++++++++ .../gb200/mxfp8_256GPU_TP1PP4EP64.yaml | 271 ++++++++++++++++++ .../gb300/mxfp8_256GPU_TP1PP4EP64.yaml | 262 +++++++++++++++++ .../h100/bf16_1024GPU_TP1PP16EP64.yaml | 192 +++++++++++++ .../h100/fp8_1024GPU_TP2PP8EP64.yaml | 207 +++++++++++++ ...TP1PP1EP64_paged_stash_fullcg_overlap.yaml | 259 +++++++++++++++++ ..._128GPU_TP1PP1EP64_partial_cg_overlap.yaml | 219 ++++++++++++++ ...128GPU_TP1PP1EP64_paged_stash_full_cg.yaml | 247 ++++++++++++++++ .../h100/bf16_256GPU_TP2PP8EP32.yaml | 186 ++++++++++++ 9 files changed, 2059 insertions(+) create mode 100644 examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml create mode 100644 examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml create mode 100644 examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml create mode 100644 examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml create mode 100644 examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml create mode 100644 examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml create mode 100644 examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml create mode 100644 examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml create mode 100644 examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml diff --git a/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml b/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml new file mode 100644 index 00000000000..0979f7ad6ab --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml @@ -0,0 +1,216 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: b300-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, cuDNN frontend, and FLA for + # Qwen3.5 Gated Delta Net support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" && \ + pip install --no-cache-dir fla-core==0.4.2 flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 8 + expert_model_parallel_size: 32 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: deepep + moe_deepep_num_sms: 24 + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + recompute_granularity: selective + recompute_modules: + - mla_up_proj + - mlp + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 2048 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: mxfp8 + fp8_format: e4m3 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_quantization: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-B200-MXFP8-TP1PP8EP32-GBS2048 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml b/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml new file mode 100644 index 00000000000..9f81ca049b7 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml @@ -0,0 +1,271 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE commit 2026-04-22 + ARG TE_COMMIT="0be9046db16d75bd41301750949a928aa667b47c" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which current TE/MCore still pass. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # Install DeepEP (single-node mode), matching upstream megatron-moe-scripts + # Dockerfile.gb. The multi-node HybridEP path (HYBRID_EP_MULTINODE=1) currently + # fails to build because DeepEP@1b8f4679 pulls in nixl.h, so build this image + # with --target base to skip the hybridep stage below. + ARG HEP_COMMIT="1b8f467965bb818bf2f6511e06993f5607e1721f" + RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git /workspace/DeepEP && \ + cd /workspace/DeepEP && git checkout ${HEP_COMMIT} && \ + TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . && \ + rm -rf /root/.cache /tmp/* + + # ========================= + # Option 1: HybridEP (multi-node) — currently broken: DeepEP@1b8f4679 pulls in + # nixl.h, which isn't available in this container. Kept for reference; do NOT + # build with `--target hybridep` until NIXL is staged into the build context. + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP commit pinned (2026-04-16) + cd DeepEP && git checkout 1b8f467965bb818bf2f6511e06993f5607e1721f + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + NCCL_GRAPH_REGISTER: 0 + NVTE_CPU_OFFLOAD_V1: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + NUM_OF_STAGES_DISPATCH_API: '10' + NUM_OF_IN_FLIGHT_S2G_DISPATCH_API: '8' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 4 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_hybridep_num_sms: 32 + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + recompute_granularity: selective + recompute_modules: + - mla_up_proj + offload_optimizer_states: true + fine_grained_activation_offloading: true + offload_modules: + - expert_fc1 + delay_offload_until_cuda_graph: true + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + te_rng_tracker: true + cuda_graph_warmup_steps: 1 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: mxfp8 + fp8_format: e4m3 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_quantization: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-GB200-MXFP8-TP1PP4EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml b/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml new file mode 100644 index 00000000000..6727da78bc7 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml @@ -0,0 +1,262 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE commit 2026-04-22 + ARG TE_COMMIT="0be9046db16d75bd41301750949a928aa667b47c" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which current TE/MCore still pass. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # Install DeepEP (single-node mode), matching upstream megatron-moe-scripts + # Dockerfile.gb. The multi-node HybridEP path (HYBRID_EP_MULTINODE=1) currently + # fails to build because DeepEP@1b8f4679 pulls in nixl.h, so build this image + # with --target base to skip the hybridep stage below. + ARG HEP_COMMIT="1b8f467965bb818bf2f6511e06993f5607e1721f" + RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git /workspace/DeepEP && \ + cd /workspace/DeepEP && git checkout ${HEP_COMMIT} && \ + TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . && \ + rm -rf /root/.cache /tmp/* + + # ========================= + # Option 1: HybridEP (multi-node) — currently broken: DeepEP@1b8f4679 pulls in + # nixl.h, which isn't available in this container. Kept for reference; do NOT + # build with `--target hybridep` until NIXL is staged into the build context. + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP commit pinned (2026-04-16) + cd DeepEP && git checkout 1b8f467965bb818bf2f6511e06993f5607e1721f + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: 0 + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + NUM_OF_STAGES_DISPATCH_API: '10' + NUM_OF_IN_FLIGHT_S2G_DISPATCH_API: '8' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 4 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_hybridep_num_sms: 32 + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + te_rng_tracker: true + cuda_graph_warmup_steps: 1 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: mxfp8 + fp8_format: e4m3 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_quantization: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-GB300-MXFP8-TP1PP4EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml b/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml new file mode 100644 index 00000000000..bc35ea5f510 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml @@ -0,0 +1,192 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 16 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + overlap_grad_reduce: true + overlap_param_gather: true + moe_token_dispatcher_type: flex + moe_enable_deepep: true + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-TP1PP16EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml b/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml new file mode 100644 index 00000000000..9fcc2696c96 --- /dev/null +++ b/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml @@ -0,0 +1,207 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 61 + hidden_size: 7168 + ffn_hidden_size: 18432 + num_attention_heads: 128 + kv_channels: 128 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1536 + kv_lora_rank: 512 + qk_head_dim: 128 + qk_pos_emb_head_dim: 64 + v_head_dim: 128 + rotary_scaling_factor: 40 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + num_experts: 256 + moe_layer_freq: ([0]*3+[1]*58) + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: true + moe_aux_loss_coeff: 1e-4 + moe_router_group_topk: 4 + moe_router_num_groups: 8 + moe_router_topk_scaling_factor: 2.5 + moe_router_score_function: sigmoid + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 2 + pipeline_model_parallel_size: 8 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + pipeline_model_parallel_layout: Et*3|(tt|)*29m|L + use_distributed_optimizer: true + sequence_parallel: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: deepep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + recompute_granularity: selective + recompute_modules: + - mla_up_proj + - mlp + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: blockwise + fp8_format: e4m3 + fp8_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_fp8: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V3-FP8-TP2PP8EP64-GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml new file mode 100644 index 00000000000..a9581dc95e8 --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml @@ -0,0 +1,259 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.04-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604-te215 + # + # Variant of gb200-gb300-2604 with TransformerEngine main @ v2.15 and + # cuDNN-frontend bumped to >=1.23.0. Otherwise identical to + # Dockerfile.GB200.GB300.torch2604. + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE v2.15 main + ARG TE_COMMIT="6cdd7115e1c1e6605feaff6ebe111ea0466cd71a" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support + # NOTE: nvidia-cudnn-frontend>=1.23.0 (TE v2.15 CuTe DSL fused grouped MLP / + # grouped_gemm_glu_wrapper_sm100). Pin cutlass-dsl>=4.4.2 for cutlass module compatibility. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]>=4.4.2" && \ + pip install --no-cache-dir "nvidia-cudnn-frontend[cutedsl]>=1.23.0" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP latest hybrid-ep tip (DOCA path; older 1b8f4679 forced NIXL via .use_nixl marker) + cd DeepEP && git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6 + rm -f .use_nixl + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 USE_NIXL=0 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True + TORCH_NCCL_AVOID_RECORD_STREAMS: '0' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: '0' + NCCL_DEBUG: VERSION + NVTE_FUSED_ATTN: '1' + NVLINK_DOMAIN_SIZE: '72' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + USE_TE_OPS: 'True' + CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL: 'True' + NVTE_NORM_FWD_USE_CUDNN: '0' + NVTE_NORM_BWD_USE_CUDNN: '0' + NVTE_FWD_LAYERNORM_SM_MARGIN: '20' + NVTE_BWD_LAYERNORM_SM_MARGIN: '20' + NVTE_CPU_OFFLOAD_V1: '1' + NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: '128' + NUM_OF_TOKENS_PER_CHUNK_DISPATCH_API: '128' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + PYTHONWARNINGS: ignore + NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_hybridep_num_sms: 32 + moe_paged_stash: true + moe_expert_rank_capacity_factor: 1.2 + moe_paged_stash_page_size: 64 + moe_paged_stash_buffer_size_factor_cuda: 1.0 + moe_paged_stash_buffer_size_factor_cpu: 0.8 + use_transformer_engine_op_fuser: true + moe_pad_experts_for_cuda_graph_inference: true + moe_mlp_glu_interleave_size: 32 + cuda_graph_impl: local + cuda_graph_scope: full_iteration + cuda_graph_warmup_steps: 2 + no_check_for_nan_in_loss_and_grad: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + te_rng_tracker: true + cross_entropy_fusion_impl: te + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 268554688 + exit_duration_in_mins: 230 + distributed_timeout_minutes: 220 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save: ${OUTPUT_PATH}/checkpoints + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_project: ${WANDB_PROJECT} + wandb_exp_name: Qwen3-235B-GB200-MXFP8-PagedStash-FullCG-Overlap-TP1PP1EP64-MBS1GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml new file mode 100644 index 00000000000..89f1bab705d --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml @@ -0,0 +1,219 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True + TORCH_NCCL_AVOID_RECORD_STREAMS: '0' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: '0' + NCCL_DEBUG: VERSION + NVTE_FUSED_ATTN: '1' + NVLINK_DOMAIN_SIZE: '72' + NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64' + USE_MNNVL: '1' + USE_TE_OPS: 'True' + CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL: 'True' + NVTE_NORM_FWD_USE_CUDNN: '0' + NVTE_NORM_BWD_USE_CUDNN: '0' + NVTE_FWD_LAYERNORM_SM_MARGIN: '20' + NVTE_BWD_LAYERNORM_SM_MARGIN: '20' + NVTE_CPU_OFFLOAD_V1: '1' + NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: '128' + NUM_OF_TOKENS_PER_CHUNK_DISPATCH_API: '128' + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + PYTHONWARNINGS: ignore +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_hybridep_num_sms: 32 + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + cuda_graph_warmup_steps: 2 + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + te_rng_tracker: true + cross_entropy_fusion_impl: te + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_samples: 268554688 + exit_duration_in_mins: 230 + distributed_timeout_minutes: 220 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save: ${OUTPUT_PATH}/checkpoints + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_project: ${WANDB_PROJECT} + wandb_exp_name: Qwen3-235B-GB200-MXFP8-PartialCG-Overlap-TP1PP1EP64-MBS1GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml b/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml new file mode 100644 index 00000000000..2b2d42e4494 --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml @@ -0,0 +1,247 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.04-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-gb300-2604-te215 + # + # Variant of gb200-gb300-2604 with TransformerEngine main @ v2.15 and + # cuDNN-frontend bumped to >=1.23.0. Otherwise identical to + # Dockerfile.GB200.GB300.torch2604. + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + # TE v2.15 main + ARG TE_COMMIT="6cdd7115e1c1e6605feaff6ebe111ea0466cd71a" + # 100a = SM100a (GB200), 103a = SM103a (GB300) + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo. + # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64. + RUN bash -ex <<"EOF" + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") REPO_ARCH=x86_64 ;; + "aarch64") REPO_ARCH=sbsa ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + apt-get update + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb + dpkg -i cuda-keyring_1.1-1_all.deb + rm cuda-keyring_1.1-1_all.deb + apt-get update + apt-get install -y --no-install-recommends libcudnn9-cuda-13 + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support + # NOTE: nvidia-cudnn-frontend>=1.23.0 (TE v2.15 CuTe DSL fused grouped MLP / + # grouped_gemm_glu_wrapper_sm100). Pin cutlass-dsl>=4.4.2 for cutlass module compatibility. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]>=4.4.2" && \ + pip install --no-cache-dir "nvidia-cudnn-frontend[cutedsl]>=1.23.0" + + # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds) + RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \ + CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \ + pip install --no-cache-dir --no-build-isolation \ + git+https://github.com/deepseek-ai/FlashMLA.git + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + # HybridEP latest hybrid-ep tip (DOCA path; older 1b8f4679 forced NIXL via .use_nixl marker) + cd DeepEP && git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6 + rm -f .use_nixl + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 USE_NIXL=0 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: graph_capture_record_stream_reuse:True + TORCH_NCCL_AVOID_RECORD_STREAMS: '0' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: '0' + NCCL_DEBUG: VERSION + NVTE_FUSED_ATTN: '1' + USE_MNNVL: '1' + NVTE_NORM_FWD_USE_CUDNN: '0' + NVTE_NORM_BWD_USE_CUDNN: '0' + NVTE_FWD_LAYERNORM_SM_MARGIN: '20' + NVTE_BWD_LAYERNORM_SM_MARGIN: '20' + NVTE_CPU_OFFLOAD_V1: '1' + PYTHONWARNINGS: ignore + NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_hybridep_num_sms: 32 + moe_paged_stash: true + moe_expert_rank_capacity_factor: 1.2 + moe_paged_stash_page_size: 64 + moe_paged_stash_buffer_size_factor_cuda: 1.0 + use_transformer_engine_op_fuser: true + moe_pad_experts_for_cuda_graph_inference: true + moe_mlp_glu_interleave_size: 32 + cuda_graph_impl: local + cuda_graph_scope: full_iteration + cuda_graph_warmup_steps: 2 + no_check_for_nan_in_loss_and_grad: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + te_rng_tracker: true + cross_entropy_fusion_impl: te + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 8192 + train_iters: 20 + distributed_timeout_minutes: 220 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save: ${OUTPUT_PATH}/checkpoints + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_project: ${WANDB_PROJECT} + wandb_exp_name: Qwen3-235B-GB300-MXFP8-PagedStash-FullCG-TP1PP1EP64-MBS1GBS8192 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml b/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml new file mode 100644 index 00000000000..d2ebd988f8d --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml @@ -0,0 +1,186 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' + NVTE_FWD_LAYERNORM_SM_MARGIN: '24' + NVTE_BWD_LAYERNORM_SM_MARGIN: '24' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 2 + pipeline_model_parallel_size: 8 + expert_model_parallel_size: 32 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + num_layers_per_virtual_pipeline_stage: 3 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + recompute_granularity: selective + recompute_modules: moe_act layernorm + cuda_graph_impl: transformer_engine + cuda_graph_scope: moe_router moe_preprocess + distributed_timeout_minutes: 220 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 2048 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-235B-TP2PP8EP32-GBS2048 + enable_experimental: true From d2867c58ef7eb38dff393d0d1f4890e2806e89c9 Mon Sep 17 00:00:00 2001 From: Dennis Liu Date: Thu, 21 May 2026 15:45:10 +0800 Subject: [PATCH 2/6] docs: add MoE recipe overview --- examples/moe_recipes/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 examples/moe_recipes/README.md diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md new file mode 100644 index 00000000000..5e8e7e7dab4 --- /dev/null +++ b/examples/moe_recipes/README.md @@ -0,0 +1,23 @@ +# MoE Recipes + +This directory contains self-contained MoE training recipes. Each YAML file includes: + +- `DEPENDENCIES`: PyTorch base image and Dockerfile content used for the recipe. +- `ENV_VARS`: Environment variables expected by the runtime. +- `ARGS`: Megatron-LM training arguments. + +## Recipe Overview + +| Model | Hardware | GPUs | Precision | Parallelism | Batch | Seq Len | PyTorch Base | Key Features | Recipe | +|-------|----------|------|-----------|-------------|-------|---------|--------------|--------------|--------| +| DeepSeek-V3 | B200 | 256 | MXFP8 | TP1 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | DeepEP dispatcher, MoE overlap | [`deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml`](deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml) | +| DeepSeek-V3 | GB200 | 256 | MXFP8 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | HybridEP, partial CUDA graph, MoE overlap, activation offload | [`deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml`](deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml) | +| DeepSeek-V3 | GB300 | 256 | MXFP8 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | HybridEP, partial CUDA graph, MoE overlap | [`deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml`](deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml) | +| DeepSeek-V3 | H100 | 1024 | BF16 | TP1 PP16 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | Baseline BF16 large-scale recipe | [`deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml`](deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml) | +| DeepSeek-V3 | H100 | 1024 | FP8 | TP2 PP8 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | DeepEP dispatcher, MoE overlap | [`deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml`](deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml) | +| Qwen3-235B-A22B | GB200 | 128 | MXFP8 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.04-py3` | Paged stash, full CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml`](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml) | +| Qwen3-235B-A22B | GB200 | 128 | MXFP8 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | Partial CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml`](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml) | +| Qwen3-235B-A22B | GB300 | 128 | MXFP8 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.04-py3` | Paged stash, full CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml`](qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml) | +| Qwen3-235B-A22B | H100 | 256 | BF16 | TP2 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | Router/preprocess CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml`](qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml) | + +Legend: TP = tensor parallel, PP = pipeline parallel, EP = expert parallel, CP = context parallel, ETP = expert tensor parallel, MBS = micro-batch size, GBS = global batch size. From 4c7acd6c97c997311f427619caa8f3935b8f2041 Mon Sep 17 00:00:00 2001 From: Dennis Liu Date: Thu, 21 May 2026 15:48:56 +0800 Subject: [PATCH 3/6] docs: improve MoE recipe README layout --- examples/moe_recipes/README.md | 47 +++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md index 5e8e7e7dab4..0e760fbb7b7 100644 --- a/examples/moe_recipes/README.md +++ b/examples/moe_recipes/README.md @@ -6,18 +6,35 @@ This directory contains self-contained MoE training recipes. Each YAML file incl - `ENV_VARS`: Environment variables expected by the runtime. - `ARGS`: Megatron-LM training arguments. -## Recipe Overview - -| Model | Hardware | GPUs | Precision | Parallelism | Batch | Seq Len | PyTorch Base | Key Features | Recipe | -|-------|----------|------|-----------|-------------|-------|---------|--------------|--------------|--------| -| DeepSeek-V3 | B200 | 256 | MXFP8 | TP1 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | DeepEP dispatcher, MoE overlap | [`deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml`](deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml) | -| DeepSeek-V3 | GB200 | 256 | MXFP8 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | HybridEP, partial CUDA graph, MoE overlap, activation offload | [`deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml`](deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml) | -| DeepSeek-V3 | GB300 | 256 | MXFP8 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | HybridEP, partial CUDA graph, MoE overlap | [`deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml`](deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml) | -| DeepSeek-V3 | H100 | 1024 | BF16 | TP1 PP16 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | Baseline BF16 large-scale recipe | [`deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml`](deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml) | -| DeepSeek-V3 | H100 | 1024 | FP8 | TP2 PP8 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | DeepEP dispatcher, MoE overlap | [`deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml`](deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml) | -| Qwen3-235B-A22B | GB200 | 128 | MXFP8 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.04-py3` | Paged stash, full CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml`](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml) | -| Qwen3-235B-A22B | GB200 | 128 | MXFP8 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | Partial CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml`](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml) | -| Qwen3-235B-A22B | GB300 | 128 | MXFP8 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 | 4096 | `nvcr.io/nvidia/pytorch:26.04-py3` | Paged stash, full CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml`](qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml) | -| Qwen3-235B-A22B | H100 | 256 | BF16 | TP2 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 | 4096 | `nvcr.io/nvidia/pytorch:26.03-py3` | Router/preprocess CUDA graph, HybridEP, MoE overlap | [`qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml`](qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml) | - -Legend: TP = tensor parallel, PP = pipeline parallel, EP = expert parallel, CP = context parallel, ETP = expert tensor parallel, MBS = micro-batch size, GBS = global batch size. +## Recipe Index + +| Recipe | GPUs | Parallelism | Batch / Seq | +|--------|------|-------------|-------------| +| [DeepSeek-V3 B200 MXFP8](deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml) | 256 | TP1 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 SL4096 | +| [DeepSeek-V3 GB200 MXFP8](deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml) | 256 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | +| [DeepSeek-V3 GB300 MXFP8](deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml) | 256 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | +| [DeepSeek-V3 H100 BF16](deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml) | 1024 | TP1 PP16 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | +| [DeepSeek-V3 H100 FP8](deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml) | 1024 | TP2 PP8 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | +| [Qwen3-235B GB200 MXFP8 full CG](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml) | 128 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | +| [Qwen3-235B GB200 MXFP8 partial CG](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml) | 128 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | +| [Qwen3-235B GB300 MXFP8 full CG](qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml) | 128 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | +| [Qwen3-235B H100 BF16](qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml) | 256 | TP2 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 SL4096 | + +## Recipe Notes + +### DeepSeek-V3 + +- **B200 MXFP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; DeepEP dispatcher and MoE overlap. +- **GB200 MXFP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; HybridEP, partial CUDA graph, MoE overlap, and activation offload. +- **GB300 MXFP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; HybridEP, partial CUDA graph, and MoE overlap. +- **H100 BF16**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; baseline large-scale BF16 recipe. +- **H100 FP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; DeepEP dispatcher and MoE overlap. + +### Qwen3-235B-A22B + +- **GB200 MXFP8 full CG**: PyTorch `nvcr.io/nvidia/pytorch:26.04-py3`; paged stash, full CUDA graph, HybridEP, and MoE overlap. +- **GB200 MXFP8 partial CG**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; partial CUDA graph, HybridEP, and MoE overlap. +- **GB300 MXFP8 full CG**: PyTorch `nvcr.io/nvidia/pytorch:26.04-py3`; paged stash, full CUDA graph, HybridEP, and MoE overlap. +- **H100 BF16**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; router/preprocess CUDA graph, HybridEP, and MoE overlap. + +Legend: TP = tensor parallel, PP = pipeline parallel, EP = expert parallel, CP = context parallel, ETP = expert tensor parallel, MBS = micro-batch size, GBS = global batch size, SL = sequence length, CG = CUDA graph. From 7ef58c5c0b31a732a426640a710c3df5bca94788 Mon Sep 17 00:00:00 2001 From: Dennis Liu Date: Thu, 21 May 2026 15:51:28 +0800 Subject: [PATCH 4/6] docs: group MoE recipes by model --- examples/moe_recipes/README.md | 80 +++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md index 0e760fbb7b7..a14bffa7a0c 100644 --- a/examples/moe_recipes/README.md +++ b/examples/moe_recipes/README.md @@ -8,17 +8,75 @@ This directory contains self-contained MoE training recipes. Each YAML file incl ## Recipe Index -| Recipe | GPUs | Parallelism | Batch / Seq | -|--------|------|-------------|-------------| -| [DeepSeek-V3 B200 MXFP8](deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml) | 256 | TP1 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 SL4096 | -| [DeepSeek-V3 GB200 MXFP8](deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml) | 256 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | -| [DeepSeek-V3 GB300 MXFP8](deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml) | 256 | TP1 PP4 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | -| [DeepSeek-V3 H100 BF16](deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml) | 1024 | TP1 PP16 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | -| [DeepSeek-V3 H100 FP8](deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml) | 1024 | TP2 PP8 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | -| [Qwen3-235B GB200 MXFP8 full CG](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml) | 128 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | -| [Qwen3-235B GB200 MXFP8 partial CG](qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml) | 128 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | -| [Qwen3-235B GB300 MXFP8 full CG](qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml) | 128 | TP1 PP1 EP64 CP1 ETP1 | MBS1 GBS8192 SL4096 | -| [Qwen3-235B H100 BF16](qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml) | 256 | TP2 PP8 EP32 CP1 ETP1 | MBS1 GBS2048 SL4096 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelRecipeGPUsParallelismBatch / Seq
DeepSeek-V3B200 MXFP8256TP1 PP8 EP32 CP1 ETP1MBS1 GBS2048 SL4096
GB200 MXFP8256TP1 PP4 EP64 CP1 ETP1MBS1 GBS8192 SL4096
GB300 MXFP8256TP1 PP4 EP64 CP1 ETP1MBS1 GBS8192 SL4096
H100 BF161024TP1 PP16 EP64 CP1 ETP1MBS1 GBS8192 SL4096
H100 FP81024TP2 PP8 EP64 CP1 ETP1MBS1 GBS8192 SL4096
Qwen3-235B-A22BGB200 MXFP8 full CG128TP1 PP1 EP64 CP1 ETP1MBS1 GBS8192 SL4096
GB200 MXFP8 partial CG128TP1 PP1 EP64 CP1 ETP1MBS1 GBS8192 SL4096
GB300 MXFP8 full CG128TP1 PP1 EP64 CP1 ETP1MBS1 GBS8192 SL4096
H100 BF16256TP2 PP8 EP32 CP1 ETP1MBS1 GBS2048 SL4096
## Recipe Notes From f5dda6e79258fb7b0ccbf3b085038dcd473e42e1 Mon Sep 17 00:00:00 2001 From: Dennis Liu Date: Thu, 21 May 2026 15:55:32 +0800 Subject: [PATCH 5/6] docs: compact MoE recipe table --- examples/moe_recipes/README.md | 69 +++++++++++++++------------------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md index a14bffa7a0c..bdb4496b216 100644 --- a/examples/moe_recipes/README.md +++ b/examples/moe_recipes/README.md @@ -14,8 +14,9 @@ This directory contains self-contained MoE training recipes. Each YAML file incl Model Recipe GPUs - Parallelism - Batch / Seq + TP/PP/EP/CP/ETP + MBS/GBS/SL + Features @@ -23,76 +24,68 @@ This directory contains self-contained MoE training recipes. Each YAML file incl DeepSeek-V3 B200 MXFP8 256 - TP1 PP8 EP32 CP1 ETP1 - MBS1 GBS2048 SL4096 + 1/8/32/1/1 + 1/2048/4096 + DeepEP
MoE overlap GB200 MXFP8 256 - TP1 PP4 EP64 CP1 ETP1 - MBS1 GBS8192 SL4096 + 1/4/64/1/1 + 1/8192/4096 + HybridEP
partial CG
MoE overlap
offload GB300 MXFP8 256 - TP1 PP4 EP64 CP1 ETP1 - MBS1 GBS8192 SL4096 + 1/4/64/1/1 + 1/8192/4096 + HybridEP
partial CG
MoE overlap H100 BF16 1024 - TP1 PP16 EP64 CP1 ETP1 - MBS1 GBS8192 SL4096 + 1/16/64/1/1 + 1/8192/4096 + BF16 baseline H100 FP8 1024 - TP2 PP8 EP64 CP1 ETP1 - MBS1 GBS8192 SL4096 + 2/8/64/1/1 + 1/8192/4096 + DeepEP
MoE overlap Qwen3-235B-A22B GB200 MXFP8 full CG 128 - TP1 PP1 EP64 CP1 ETP1 - MBS1 GBS8192 SL4096 + 1/1/64/1/1 + 1/8192/4096 + paged stash
full CG
HybridEP
MoE overlap GB200 MXFP8 partial CG 128 - TP1 PP1 EP64 CP1 ETP1 - MBS1 GBS8192 SL4096 + 1/1/64/1/1 + 1/8192/4096 + partial CG
HybridEP
MoE overlap GB300 MXFP8 full CG 128 - TP1 PP1 EP64 CP1 ETP1 - MBS1 GBS8192 SL4096 + 1/1/64/1/1 + 1/8192/4096 + paged stash
full CG
HybridEP
MoE overlap H100 BF16 256 - TP2 PP8 EP32 CP1 ETP1 - MBS1 GBS2048 SL4096 + 2/8/32/1/1 + 1/2048/4096 + router/preprocess CG
HybridEP
MoE overlap -## Recipe Notes - -### DeepSeek-V3 - -- **B200 MXFP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; DeepEP dispatcher and MoE overlap. -- **GB200 MXFP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; HybridEP, partial CUDA graph, MoE overlap, and activation offload. -- **GB300 MXFP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; HybridEP, partial CUDA graph, and MoE overlap. -- **H100 BF16**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; baseline large-scale BF16 recipe. -- **H100 FP8**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; DeepEP dispatcher and MoE overlap. - -### Qwen3-235B-A22B - -- **GB200 MXFP8 full CG**: PyTorch `nvcr.io/nvidia/pytorch:26.04-py3`; paged stash, full CUDA graph, HybridEP, and MoE overlap. -- **GB200 MXFP8 partial CG**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; partial CUDA graph, HybridEP, and MoE overlap. -- **GB300 MXFP8 full CG**: PyTorch `nvcr.io/nvidia/pytorch:26.04-py3`; paged stash, full CUDA graph, HybridEP, and MoE overlap. -- **H100 BF16**: PyTorch `nvcr.io/nvidia/pytorch:26.03-py3`; router/preprocess CUDA graph, HybridEP, and MoE overlap. - -Legend: TP = tensor parallel, PP = pipeline parallel, EP = expert parallel, CP = context parallel, ETP = expert tensor parallel, MBS = micro-batch size, GBS = global batch size, SL = sequence length, CG = CUDA graph. +Legend: TP = tensor parallel, PP = pipeline parallel, EP = expert parallel, CP = context parallel, ETP = expert tensor parallel, MBS = micro-batch size, GBS = global batch size, SL = sequence length, CG = CUDA graph. The tuple orders are `TP/PP/EP/CP/ETP` and `MBS/GBS/SL`. From 29686bd490180ad9cf2df9b61bd0d50768e362e1 Mon Sep 17 00:00:00 2001 From: Dennis Liu Date: Thu, 21 May 2026 16:19:23 +0800 Subject: [PATCH 6/6] docs: refine MoE recipe feature labels --- examples/moe_recipes/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md index bdb4496b216..f77f6fad115 100644 --- a/examples/moe_recipes/README.md +++ b/examples/moe_recipes/README.md @@ -26,21 +26,21 @@ This directory contains self-contained MoE training recipes. Each YAML file incl 256 1/8/32/1/1 1/2048/4096 - DeepEP
MoE overlap + DeepEP; EP overlap GB200 MXFP8 256 1/4/64/1/1 1/8192/4096 - HybridEP
partial CG
MoE overlap
offload + HybridEP; partial CG; EP overlap; offload GB300 MXFP8 256 1/4/64/1/1 1/8192/4096 - HybridEP
partial CG
MoE overlap + HybridEP; partial CG; EP overlap H100 BF16 @@ -54,7 +54,7 @@ This directory contains self-contained MoE training recipes. Each YAML file incl 1024 2/8/64/1/1 1/8192/4096 - DeepEP
MoE overlap + DeepEP; EP overlap Qwen3-235B-A22B @@ -62,28 +62,28 @@ This directory contains self-contained MoE training recipes. Each YAML file incl 128 1/1/64/1/1 1/8192/4096 - paged stash
full CG
HybridEP
MoE overlap + paged stash; full CG; HybridEP; EP overlap GB200 MXFP8 partial CG 128 1/1/64/1/1 1/8192/4096 - partial CG
HybridEP
MoE overlap + partial CG; HybridEP; EP overlap GB300 MXFP8 full CG 128 1/1/64/1/1 1/8192/4096 - paged stash
full CG
HybridEP
MoE overlap + paged stash; full CG; HybridEP; EP overlap H100 BF16 256 2/8/32/1/1 1/2048/4096 - router/preprocess CG
HybridEP
MoE overlap + router/preprocess CG; HybridEP; EP overlap