diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md
new file mode 100644
index 00000000000..f77f6fad115
--- /dev/null
+++ b/examples/moe_recipes/README.md
@@ -0,0 +1,91 @@
+# MoE Recipes
+
+This directory contains self-contained MoE training recipes. Each YAML file includes:
+
+- `DEPENDENCIES`: PyTorch base image and Dockerfile content used for the recipe.
+- `ENV_VARS`: Environment variables expected by the runtime.
+- `ARGS`: Megatron-LM training arguments.
+
+## Recipe Index
+
+
+
+
+ | Model |
+ Recipe |
+ GPUs |
+ TP/PP/EP/CP/ETP |
+ MBS/GBS/SL |
+ Features |
+
+
+
+
+ | DeepSeek-V3 |
+ B200 MXFP8 |
+ 256 |
+ 1/8/32/1/1 |
+ 1/2048/4096 |
+ DeepEP; EP overlap |
+
+
+ | GB200 MXFP8 |
+ 256 |
+ 1/4/64/1/1 |
+ 1/8192/4096 |
+ HybridEP; partial CG; EP overlap; offload |
+
+
+ | GB300 MXFP8 |
+ 256 |
+ 1/4/64/1/1 |
+ 1/8192/4096 |
+ HybridEP; partial CG; EP overlap |
+
+
+ | H100 BF16 |
+ 1024 |
+ 1/16/64/1/1 |
+ 1/8192/4096 |
+ BF16 baseline |
+
+
+ | H100 FP8 |
+ 1024 |
+ 2/8/64/1/1 |
+ 1/8192/4096 |
+ DeepEP; EP overlap |
+
+
+ | Qwen3-235B-A22B |
+ GB200 MXFP8 full CG |
+ 128 |
+ 1/1/64/1/1 |
+ 1/8192/4096 |
+ paged stash; full CG; HybridEP; EP overlap |
+
+
+ | GB200 MXFP8 partial CG |
+ 128 |
+ 1/1/64/1/1 |
+ 1/8192/4096 |
+ partial CG; HybridEP; EP overlap |
+
+
+ | GB300 MXFP8 full CG |
+ 128 |
+ 1/1/64/1/1 |
+ 1/8192/4096 |
+ paged stash; full CG; HybridEP; EP overlap |
+
+
+ | H100 BF16 |
+ 256 |
+ 2/8/32/1/1 |
+ 1/2048/4096 |
+ router/preprocess CG; HybridEP; EP overlap |
+
+
+
+
+Legend: TP = tensor parallel, PP = pipeline parallel, EP = expert parallel, CP = context parallel, ETP = expert tensor parallel, MBS = micro-batch size, GBS = global batch size, SL = sequence length, CG = CUDA graph. The tuple orders are `TP/PP/EP/CP/ETP` and `MBS/GBS/SL`.
diff --git a/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml b/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml
new file mode 100644
index 00000000000..0979f7ad6ab
--- /dev/null
+++ b/examples/moe_recipes/deepseek_v3/b200/mxfp8_256GPU_TP1PP8EP32.yaml
@@ -0,0 +1,216 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: b300-torch2603
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ ARG TE_TAG=release_v2.14
+ ARG NVTE_CUDA_ARCHS="100a;103a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml omegaconf \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}"
+
+ # Install Flash Attention 4, CUTLASS DSL, cuDNN frontend, and FLA for
+ # Qwen3.5 Gated Delta Net support.
+ # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from
+ # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes.
+ RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \
+ pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" && \
+ pip install --no-cache-dir fla-core==0.4.2 flash-linear-attention==0.4.2
+
+ # =========================
+ # Option 1: HybridEP
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \
+ TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: '0'
+ NVTE_FUSED_ATTN: '1'
+ NVTE_NORM_FWD_USE_CUDNN: '1'
+ NVTE_NORM_BWD_USE_CUDNN: '1'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '24'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '24'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: unsloth/DeepSeek-V3
+ num_layers: 61
+ hidden_size: 7168
+ ffn_hidden_size: 18432
+ num_attention_heads: 128
+ kv_channels: 128
+ max_position_embeddings: 4096
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_base: 10000
+ make_vocab_size_divisible_by: 3232
+ multi_latent_attention: true
+ q_lora_rank: 1536
+ kv_lora_rank: 512
+ qk_head_dim: 128
+ qk_pos_emb_head_dim: 64
+ v_head_dim: 128
+ rotary_scaling_factor: 40
+ mscale: 1.0
+ mscale_all_dim: 1.0
+ qk_layernorm: true
+ num_experts: 256
+ moe_layer_freq: ([0]*3+[1]*58)
+ moe_ffn_hidden_size: 2048
+ moe_shared_expert_intermediate_size: 2048
+ moe_router_load_balancing_type: seq_aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: true
+ moe_aux_loss_coeff: 1e-4
+ moe_router_group_topk: 4
+ moe_router_num_groups: 8
+ moe_router_topk_scaling_factor: 2.5
+ moe_router_score_function: sigmoid
+ moe_router_enable_expert_bias: true
+ moe_router_bias_update_rate: 1e-3
+ mtp_num_layers: 1
+ mtp_loss_scaling_factor: 0.1
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 1
+ pipeline_model_parallel_size: 8
+ expert_model_parallel_size: 32
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: deepep
+ moe_deepep_num_sms: 24
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ recompute_granularity: selective
+ recompute_modules:
+ - mla_up_proj
+ - mlp
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 2048
+ train_samples: 585937500
+ exit_duration_in_mins: 220
+ no_save_optim: true
+ no_check_for_nan_in_loss_and_grad: true
+ cross_entropy_loss_fusion: true
+ cross_entropy_fusion_impl: te
+ no_create_attention_mask_in_dataloader: true
+ manual_gc: true
+ manual_gc_interval: 10
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ fp8_recipe: mxfp8
+ fp8_format: e4m3
+ fp8_param_gather: true
+ reuse_grad_buf_for_mxfp8_param_ag: true
+ overlap_grad_reduce: true
+ overlap_param_gather: true
+ use_precision_aware_optimizer: true
+ main_grads_dtype: fp32
+ main_params_dtype: fp32
+ exp_avg_dtype: bf16
+ exp_avg_sq_dtype: bf16
+ moe_router_padding_for_quantization: true
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 200
+ finetune: false
+ no_load_optim: true
+ no_load_rng: true
+ auto_detect_ckpt_format: true
+ load: ${LOAD_PATH}
+ save_interval: 500
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_exp_name: DeepSeek-V3-B200-MXFP8-TP1PP8EP32-GBS2048
+ enable_experimental: true
diff --git a/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml b/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml
new file mode 100644
index 00000000000..9f81ca049b7
--- /dev/null
+++ b/examples/moe_recipes/deepseek_v3/gb200/mxfp8_256GPU_TP1PP4EP64.yaml
@@ -0,0 +1,271 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: gb200-gb300-2604
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ # TE commit 2026-04-22
+ ARG TE_COMMIT="0be9046db16d75bd41301750949a928aa667b47c"
+ # 100a = SM100a (GB200), 103a = SM103a (GB300)
+ ARG NVTE_CUDA_ARCHS="100a;103a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml omegaconf \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo.
+ # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64.
+ RUN bash -ex <<"EOF"
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") REPO_ARCH=x86_64 ;;
+ "aarch64") REPO_ARCH=sbsa ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ apt-get update
+ wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb
+ dpkg -i cuda-keyring_1.1-1_all.deb
+ rm cuda-keyring_1.1-1_all.deb
+ apt-get update
+ apt-get install -y --no-install-recommends libcudnn9-cuda-13
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}"
+
+ # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support.
+ # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from
+ # grouped_gemm_quant_wrapper_sm100, which current TE/MCore still pass.
+ RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]==4.4.2" && \
+ pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1"
+
+ # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds)
+ RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \
+ CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \
+ pip install --no-cache-dir --no-build-isolation \
+ git+https://github.com/deepseek-ai/FlashMLA.git
+
+ # Install DeepEP (single-node mode), matching upstream megatron-moe-scripts
+ # Dockerfile.gb. The multi-node HybridEP path (HYBRID_EP_MULTINODE=1) currently
+ # fails to build because DeepEP@1b8f4679 pulls in nixl.h, so build this image
+ # with --target base to skip the hybridep stage below.
+ ARG HEP_COMMIT="1b8f467965bb818bf2f6511e06993f5607e1721f"
+ RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git /workspace/DeepEP && \
+ cd /workspace/DeepEP && git checkout ${HEP_COMMIT} && \
+ TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . && \
+ rm -rf /root/.cache /tmp/*
+
+ # =========================
+ # Option 1: HybridEP (multi-node) — currently broken: DeepEP@1b8f4679 pulls in
+ # nixl.h, which isn't available in this container. Kept for reference; do NOT
+ # build with `--target hybridep` until NIXL is staged into the build context.
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ # HybridEP commit pinned (2026-04-16)
+ cd DeepEP && git checkout 1b8f467965bb818bf2f6511e06993f5607e1721f
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \
+ TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: '0'
+ NVTE_FUSED_ATTN: '1'
+ NVTE_NORM_FWD_USE_CUDNN: '1'
+ NVTE_NORM_BWD_USE_CUDNN: '1'
+ NCCL_GRAPH_REGISTER: 0
+ NVTE_CPU_OFFLOAD_V1: '1'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '24'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '24'
+ NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64'
+ USE_MNNVL: '1'
+ NUM_OF_STAGES_DISPATCH_API: '10'
+ NUM_OF_IN_FLIGHT_S2G_DISPATCH_API: '8'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: unsloth/DeepSeek-V3
+ num_layers: 61
+ hidden_size: 7168
+ ffn_hidden_size: 18432
+ num_attention_heads: 128
+ kv_channels: 128
+ max_position_embeddings: 4096
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_base: 10000
+ make_vocab_size_divisible_by: 3232
+ multi_latent_attention: true
+ q_lora_rank: 1536
+ kv_lora_rank: 512
+ qk_head_dim: 128
+ qk_pos_emb_head_dim: 64
+ v_head_dim: 128
+ rotary_scaling_factor: 40
+ mscale: 1.0
+ mscale_all_dim: 1.0
+ qk_layernorm: true
+ num_experts: 256
+ moe_layer_freq: ([0]*3+[1]*58)
+ moe_ffn_hidden_size: 2048
+ moe_shared_expert_intermediate_size: 2048
+ moe_router_load_balancing_type: seq_aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: true
+ moe_aux_loss_coeff: 1e-4
+ moe_router_group_topk: 4
+ moe_router_num_groups: 8
+ moe_router_topk_scaling_factor: 2.5
+ moe_router_score_function: sigmoid
+ moe_router_enable_expert_bias: true
+ moe_router_bias_update_rate: 1e-3
+ mtp_num_layers: 1
+ mtp_loss_scaling_factor: 0.1
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 1
+ pipeline_model_parallel_size: 4
+ expert_model_parallel_size: 64
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: hybridep
+ moe_hybridep_num_sms: 32
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ recompute_granularity: selective
+ recompute_modules:
+ - mla_up_proj
+ offload_optimizer_states: true
+ fine_grained_activation_offloading: true
+ offload_modules:
+ - expert_fc1
+ delay_offload_until_cuda_graph: true
+ cuda_graph_impl: transformer_engine
+ cuda_graph_scope:
+ - attn
+ - moe_router
+ - moe_preprocess
+ te_rng_tracker: true
+ cuda_graph_warmup_steps: 1
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 8192
+ train_samples: 585937500
+ exit_duration_in_mins: 220
+ no_save_optim: true
+ no_check_for_nan_in_loss_and_grad: true
+ cross_entropy_loss_fusion: true
+ cross_entropy_fusion_impl: te
+ no_create_attention_mask_in_dataloader: true
+ manual_gc: true
+ manual_gc_interval: 10
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ fp8_recipe: mxfp8
+ fp8_format: e4m3
+ fp8_param_gather: true
+ reuse_grad_buf_for_mxfp8_param_ag: true
+ overlap_grad_reduce: true
+ overlap_param_gather: true
+ use_precision_aware_optimizer: true
+ main_grads_dtype: fp32
+ main_params_dtype: fp32
+ exp_avg_dtype: bf16
+ exp_avg_sq_dtype: bf16
+ moe_router_padding_for_quantization: true
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 200
+ finetune: false
+ no_load_optim: true
+ no_load_rng: true
+ auto_detect_ckpt_format: true
+ load: ${LOAD_PATH}
+ save_interval: 500
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_exp_name: DeepSeek-V3-GB200-MXFP8-TP1PP4EP64-GBS8192
+ enable_experimental: true
diff --git a/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml b/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml
new file mode 100644
index 00000000000..6727da78bc7
--- /dev/null
+++ b/examples/moe_recipes/deepseek_v3/gb300/mxfp8_256GPU_TP1PP4EP64.yaml
@@ -0,0 +1,262 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: gb200-gb300-2604
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ # TE commit 2026-04-22
+ ARG TE_COMMIT="0be9046db16d75bd41301750949a928aa667b47c"
+ # 100a = SM100a (GB200), 103a = SM103a (GB300)
+ ARG NVTE_CUDA_ARCHS="100a;103a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml omegaconf \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo.
+ # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64.
+ RUN bash -ex <<"EOF"
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") REPO_ARCH=x86_64 ;;
+ "aarch64") REPO_ARCH=sbsa ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ apt-get update
+ wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb
+ dpkg -i cuda-keyring_1.1-1_all.deb
+ rm cuda-keyring_1.1-1_all.deb
+ apt-get update
+ apt-get install -y --no-install-recommends libcudnn9-cuda-13
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}"
+
+ # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support.
+ # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from
+ # grouped_gemm_quant_wrapper_sm100, which current TE/MCore still pass.
+ RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]==4.4.2" && \
+ pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1"
+
+ # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds)
+ RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \
+ CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \
+ pip install --no-cache-dir --no-build-isolation \
+ git+https://github.com/deepseek-ai/FlashMLA.git
+
+ # Install DeepEP (single-node mode), matching upstream megatron-moe-scripts
+ # Dockerfile.gb. The multi-node HybridEP path (HYBRID_EP_MULTINODE=1) currently
+ # fails to build because DeepEP@1b8f4679 pulls in nixl.h, so build this image
+ # with --target base to skip the hybridep stage below.
+ ARG HEP_COMMIT="1b8f467965bb818bf2f6511e06993f5607e1721f"
+ RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git /workspace/DeepEP && \
+ cd /workspace/DeepEP && git checkout ${HEP_COMMIT} && \
+ TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . && \
+ rm -rf /root/.cache /tmp/*
+
+ # =========================
+ # Option 1: HybridEP (multi-node) — currently broken: DeepEP@1b8f4679 pulls in
+ # nixl.h, which isn't available in this container. Kept for reference; do NOT
+ # build with `--target hybridep` until NIXL is staged into the build context.
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ # HybridEP commit pinned (2026-04-16)
+ cd DeepEP && git checkout 1b8f467965bb818bf2f6511e06993f5607e1721f
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \
+ TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: '0'
+ NCCL_GRAPH_REGISTER: 0
+ NVTE_FUSED_ATTN: '1'
+ NVTE_NORM_FWD_USE_CUDNN: '1'
+ NVTE_NORM_BWD_USE_CUDNN: '1'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '24'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '24'
+ NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64'
+ USE_MNNVL: '1'
+ NUM_OF_STAGES_DISPATCH_API: '10'
+ NUM_OF_IN_FLIGHT_S2G_DISPATCH_API: '8'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: unsloth/DeepSeek-V3
+ num_layers: 61
+ hidden_size: 7168
+ ffn_hidden_size: 18432
+ num_attention_heads: 128
+ kv_channels: 128
+ max_position_embeddings: 4096
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_base: 10000
+ make_vocab_size_divisible_by: 3232
+ multi_latent_attention: true
+ q_lora_rank: 1536
+ kv_lora_rank: 512
+ qk_head_dim: 128
+ qk_pos_emb_head_dim: 64
+ v_head_dim: 128
+ rotary_scaling_factor: 40
+ mscale: 1.0
+ mscale_all_dim: 1.0
+ qk_layernorm: true
+ num_experts: 256
+ moe_layer_freq: ([0]*3+[1]*58)
+ moe_ffn_hidden_size: 2048
+ moe_shared_expert_intermediate_size: 2048
+ moe_router_load_balancing_type: seq_aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: true
+ moe_aux_loss_coeff: 1e-4
+ moe_router_group_topk: 4
+ moe_router_num_groups: 8
+ moe_router_topk_scaling_factor: 2.5
+ moe_router_score_function: sigmoid
+ moe_router_enable_expert_bias: true
+ moe_router_bias_update_rate: 1e-3
+ mtp_num_layers: 1
+ mtp_loss_scaling_factor: 0.1
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 1
+ pipeline_model_parallel_size: 4
+ expert_model_parallel_size: 64
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ pipeline_model_parallel_layout: Et*4|(tttt|)*14tmL
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: hybridep
+ moe_hybridep_num_sms: 32
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ cuda_graph_impl: transformer_engine
+ cuda_graph_scope:
+ - attn
+ - moe_router
+ - moe_preprocess
+ te_rng_tracker: true
+ cuda_graph_warmup_steps: 1
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 8192
+ train_samples: 585937500
+ exit_duration_in_mins: 220
+ no_save_optim: true
+ no_check_for_nan_in_loss_and_grad: true
+ cross_entropy_loss_fusion: true
+ cross_entropy_fusion_impl: te
+ no_create_attention_mask_in_dataloader: true
+ manual_gc: true
+ manual_gc_interval: 10
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ fp8_recipe: mxfp8
+ fp8_format: e4m3
+ fp8_param_gather: true
+ reuse_grad_buf_for_mxfp8_param_ag: true
+ overlap_grad_reduce: true
+ overlap_param_gather: true
+ use_precision_aware_optimizer: true
+ main_grads_dtype: fp32
+ main_params_dtype: fp32
+ exp_avg_dtype: bf16
+ exp_avg_sq_dtype: bf16
+ moe_router_padding_for_quantization: true
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 200
+ finetune: false
+ no_load_optim: true
+ no_load_rng: true
+ auto_detect_ckpt_format: true
+ load: ${LOAD_PATH}
+ save_interval: 500
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_exp_name: DeepSeek-V3-GB300-MXFP8-TP1PP4EP64-GBS8192
+ enable_experimental: true
diff --git a/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml b/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml
new file mode 100644
index 00000000000..bc35ea5f510
--- /dev/null
+++ b/examples/moe_recipes/deepseek_v3/h100/bf16_1024GPU_TP1PP16EP64.yaml
@@ -0,0 +1,192 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: h100-torch2603
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ ARG TE_TAG=release_v2.14
+ ARG NVTE_CUDA_ARCHS="90a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml omegaconf \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}"
+
+ # Install Flash Attention (standard, not FA4 which requires SM100+)
+ RUN pip install --no-cache-dir flash-attn
+
+ # =========================
+ # Option 1: HybridEP
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \
+ TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: '0'
+ NVTE_FUSED_ATTN: '1'
+ NVTE_NORM_FWD_USE_CUDNN: '1'
+ NVTE_NORM_BWD_USE_CUDNN: '1'
+ CUDA_DEVICE_MAX_CONNECTIONS: '1'
+ NCCL_GRAPH_REGISTER: '0'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: unsloth/DeepSeek-V3
+ num_layers: 61
+ hidden_size: 7168
+ ffn_hidden_size: 18432
+ num_attention_heads: 128
+ kv_channels: 128
+ max_position_embeddings: 4096
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_base: 10000
+ make_vocab_size_divisible_by: 3232
+ multi_latent_attention: true
+ q_lora_rank: 1536
+ kv_lora_rank: 512
+ qk_head_dim: 128
+ qk_pos_emb_head_dim: 64
+ v_head_dim: 128
+ rotary_scaling_factor: 40
+ mscale: 1.0
+ mscale_all_dim: 1.0
+ qk_layernorm: true
+ num_experts: 256
+ moe_layer_freq: ([0]*3+[1]*58)
+ moe_ffn_hidden_size: 2048
+ moe_shared_expert_intermediate_size: 2048
+ moe_router_load_balancing_type: seq_aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: true
+ moe_aux_loss_coeff: 1e-4
+ moe_router_group_topk: 4
+ moe_router_num_groups: 8
+ moe_router_topk_scaling_factor: 2.5
+ moe_router_score_function: sigmoid
+ moe_router_enable_expert_bias: true
+ moe_router_bias_update_rate: 1e-3
+ mtp_num_layers: 1
+ mtp_loss_scaling_factor: 0.1
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 1
+ pipeline_model_parallel_size: 16
+ expert_model_parallel_size: 64
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ overlap_grad_reduce: true
+ overlap_param_gather: true
+ moe_token_dispatcher_type: flex
+ moe_enable_deepep: true
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 8192
+ train_samples: 585937500
+ exit_duration_in_mins: 220
+ no_save_optim: true
+ no_check_for_nan_in_loss_and_grad: true
+ cross_entropy_loss_fusion: true
+ cross_entropy_fusion_impl: te
+ no_create_attention_mask_in_dataloader: true
+ manual_gc: true
+ manual_gc_interval: 10
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 200
+ finetune: false
+ no_load_optim: true
+ no_load_rng: true
+ auto_detect_ckpt_format: true
+ load: ${LOAD_PATH}
+ save_interval: 500
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_exp_name: DeepSeek-V3-TP1PP16EP64-GBS8192
+ enable_experimental: true
diff --git a/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml b/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml
new file mode 100644
index 00000000000..9fcc2696c96
--- /dev/null
+++ b/examples/moe_recipes/deepseek_v3/h100/fp8_1024GPU_TP2PP8EP64.yaml
@@ -0,0 +1,207 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: h100-torch2603
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ ARG TE_TAG=release_v2.14
+ ARG NVTE_CUDA_ARCHS="90a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml omegaconf \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}"
+
+ # Install Flash Attention (standard, not FA4 which requires SM100+)
+ RUN pip install --no-cache-dir flash-attn
+
+ # =========================
+ # Option 1: HybridEP
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \
+ TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: '0'
+ NVTE_FUSED_ATTN: '1'
+ NVTE_NORM_FWD_USE_CUDNN: '1'
+ NVTE_NORM_BWD_USE_CUDNN: '1'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '24'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '24'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: unsloth/DeepSeek-V3
+ num_layers: 61
+ hidden_size: 7168
+ ffn_hidden_size: 18432
+ num_attention_heads: 128
+ kv_channels: 128
+ max_position_embeddings: 4096
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_base: 10000
+ make_vocab_size_divisible_by: 3232
+ multi_latent_attention: true
+ q_lora_rank: 1536
+ kv_lora_rank: 512
+ qk_head_dim: 128
+ qk_pos_emb_head_dim: 64
+ v_head_dim: 128
+ rotary_scaling_factor: 40
+ mscale: 1.0
+ mscale_all_dim: 1.0
+ qk_layernorm: true
+ num_experts: 256
+ moe_layer_freq: ([0]*3+[1]*58)
+ moe_ffn_hidden_size: 2048
+ moe_shared_expert_intermediate_size: 2048
+ moe_router_load_balancing_type: seq_aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: true
+ moe_aux_loss_coeff: 1e-4
+ moe_router_group_topk: 4
+ moe_router_num_groups: 8
+ moe_router_topk_scaling_factor: 2.5
+ moe_router_score_function: sigmoid
+ moe_router_enable_expert_bias: true
+ moe_router_bias_update_rate: 1e-3
+ mtp_num_layers: 1
+ mtp_loss_scaling_factor: 0.1
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 2
+ pipeline_model_parallel_size: 8
+ expert_model_parallel_size: 64
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ pipeline_model_parallel_layout: Et*3|(tt|)*29m|L
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: deepep
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ recompute_granularity: selective
+ recompute_modules:
+ - mla_up_proj
+ - mlp
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 8192
+ train_samples: 585937500
+ exit_duration_in_mins: 220
+ no_save_optim: true
+ no_check_for_nan_in_loss_and_grad: true
+ cross_entropy_loss_fusion: true
+ cross_entropy_fusion_impl: te
+ no_create_attention_mask_in_dataloader: true
+ manual_gc: true
+ manual_gc_interval: 10
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ fp8_recipe: blockwise
+ fp8_format: e4m3
+ fp8_param_gather: true
+ use_precision_aware_optimizer: true
+ main_grads_dtype: fp32
+ main_params_dtype: fp32
+ exp_avg_dtype: bf16
+ exp_avg_sq_dtype: bf16
+ moe_router_padding_for_fp8: true
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 200
+ finetune: false
+ no_load_optim: true
+ no_load_rng: true
+ auto_detect_ckpt_format: true
+ load: ${LOAD_PATH}
+ save_interval: 500
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_exp_name: DeepSeek-V3-FP8-TP2PP8EP64-GBS8192
+ enable_experimental: true
diff --git a/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml
new file mode 100644
index 00000000000..a9581dc95e8
--- /dev/null
+++ b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_paged_stash_fullcg_overlap.yaml
@@ -0,0 +1,259 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.04-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: gb200-gb300-2604-te215
+ #
+ # Variant of gb200-gb300-2604 with TransformerEngine main @ v2.15 and
+ # cuDNN-frontend bumped to >=1.23.0. Otherwise identical to
+ # Dockerfile.GB200.GB300.torch2604.
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ # TE v2.15 main
+ ARG TE_COMMIT="6cdd7115e1c1e6605feaff6ebe111ea0466cd71a"
+ # 100a = SM100a (GB200), 103a = SM103a (GB300)
+ ARG NVTE_CUDA_ARCHS="100a;103a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo.
+ # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64.
+ RUN bash -ex <<"EOF"
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") REPO_ARCH=x86_64 ;;
+ "aarch64") REPO_ARCH=sbsa ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ apt-get update
+ wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb
+ dpkg -i cuda-keyring_1.1-1_all.deb
+ rm cuda-keyring_1.1-1_all.deb
+ apt-get update
+ apt-get install -y --no-install-recommends libcudnn9-cuda-13
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}"
+
+ # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support
+ # NOTE: nvidia-cudnn-frontend>=1.23.0 (TE v2.15 CuTe DSL fused grouped MLP /
+ # grouped_gemm_glu_wrapper_sm100). Pin cutlass-dsl>=4.4.2 for cutlass module compatibility.
+ RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]>=4.4.2" && \
+ pip install --no-cache-dir "nvidia-cudnn-frontend[cutedsl]>=1.23.0"
+
+ # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds)
+ RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \
+ CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \
+ pip install --no-cache-dir --no-build-isolation \
+ git+https://github.com/deepseek-ai/FlashMLA.git
+
+ # =========================
+ # Option 1: HybridEP
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ # HybridEP latest hybrid-ep tip (DOCA path; older 1b8f4679 forced NIXL via .use_nixl marker)
+ cd DeepEP && git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6
+ rm -f .use_nixl
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 USE_NIXL=0 \
+ TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True
+ TORCH_NCCL_AVOID_RECORD_STREAMS: '0'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NCCL_NVLS_ENABLE: '0'
+ NCCL_GRAPH_REGISTER: '0'
+ NCCL_DEBUG: VERSION
+ NVTE_FUSED_ATTN: '1'
+ NVLINK_DOMAIN_SIZE: '72'
+ NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64'
+ USE_MNNVL: '1'
+ USE_TE_OPS: 'True'
+ CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL: 'True'
+ NVTE_NORM_FWD_USE_CUDNN: '0'
+ NVTE_NORM_BWD_USE_CUDNN: '0'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '20'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '20'
+ NVTE_CPU_OFFLOAD_V1: '1'
+ NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: '128'
+ NUM_OF_TOKENS_PER_CHUNK_DISPATCH_API: '128'
+ HF_HUB_OFFLINE: '1'
+ TRANSFORMERS_OFFLINE: '1'
+ PYTHONWARNINGS: ignore
+ NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: Qwen/Qwen3-235B-A22B
+ num_layers: 94
+ hidden_size: 4096
+ ffn_hidden_size: 12288
+ num_attention_heads: 64
+ kv_channels: 128
+ max_position_embeddings: 4096
+ group_query_attention: true
+ num_query_groups: 4
+ qk_layernorm: true
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_percent: 1.0
+ rotary_base: 1000000
+ make_vocab_size_divisible_by: 1187
+ num_experts: 128
+ moe_ffn_hidden_size: 1536
+ moe_router_load_balancing_type: aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: false
+ moe_aux_loss_coeff: 1e-3
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 1
+ pipeline_model_parallel_size: 1
+ expert_model_parallel_size: 64
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ account_for_embedding_in_pipeline_split: true
+ account_for_loss_in_pipeline_split: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: hybridep
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ moe_router_padding_for_quantization: true
+ moe_hybridep_num_sms: 32
+ moe_paged_stash: true
+ moe_expert_rank_capacity_factor: 1.2
+ moe_paged_stash_page_size: 64
+ moe_paged_stash_buffer_size_factor_cuda: 1.0
+ moe_paged_stash_buffer_size_factor_cpu: 0.8
+ use_transformer_engine_op_fuser: true
+ moe_pad_experts_for_cuda_graph_inference: true
+ moe_mlp_glu_interleave_size: 32
+ cuda_graph_impl: local
+ cuda_graph_scope: full_iteration
+ cuda_graph_warmup_steps: 2
+ no_check_for_nan_in_loss_and_grad: true
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ te_rng_tracker: true
+ cross_entropy_fusion_impl: te
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 8192
+ train_samples: 268554688
+ exit_duration_in_mins: 230
+ distributed_timeout_minutes: 220
+ no_create_attention_mask_in_dataloader: true
+ cross_entropy_loss_fusion: true
+ manual_gc: true
+ manual_gc_interval: 5
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ fp8_format: e4m3
+ fp8_recipe: mxfp8
+ fp8_param_gather: true
+ reuse_grad_buf_for_mxfp8_param_ag: true
+ overlap_grad_reduce: true
+ overlap_param_gather: true
+ use_precision_aware_optimizer: true
+ main_grads_dtype: fp32
+ main_params_dtype: fp32
+ exp_avg_dtype: bf16
+ exp_avg_sq_dtype: bf16
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 500
+ finetune: true
+ auto_detect_ckpt_format: true
+ no_load_rng: true
+ no_load_optim: true
+ load: ${LOAD_PATH}
+ save: ${OUTPUT_PATH}/checkpoints
+ save_interval: 100
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_num_zeros_in_grad: true
+ log_params_norm: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_project: ${WANDB_PROJECT}
+ wandb_exp_name: Qwen3-235B-GB200-MXFP8-PagedStash-FullCG-Overlap-TP1PP1EP64-MBS1GBS8192
+ enable_experimental: true
diff --git a/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml
new file mode 100644
index 00000000000..89f1bab705d
--- /dev/null
+++ b/examples/moe_recipes/qwen3_235b/gb200/mxfp8_128GPU_TP1PP1EP64_partial_cg_overlap.yaml
@@ -0,0 +1,219 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: gb200-torch2603
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ ARG TE_TAG=release_v2.14
+ ARG NVTE_CUDA_ARCHS="100a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml omegaconf \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}"
+
+ # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support.
+ # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from
+ # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes.
+ RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \
+ pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1"
+
+ # =========================
+ # Option 1: HybridEP
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \
+ TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True
+ TORCH_NCCL_AVOID_RECORD_STREAMS: '0'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NCCL_NVLS_ENABLE: '0'
+ NCCL_GRAPH_REGISTER: '0'
+ NCCL_DEBUG: VERSION
+ NVTE_FUSED_ATTN: '1'
+ NVLINK_DOMAIN_SIZE: '72'
+ NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: '64'
+ USE_MNNVL: '1'
+ USE_TE_OPS: 'True'
+ CUDNN_FE_GROUPED_GEMM_DYNAMIC_MNKL: 'True'
+ NVTE_NORM_FWD_USE_CUDNN: '0'
+ NVTE_NORM_BWD_USE_CUDNN: '0'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '20'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '20'
+ NVTE_CPU_OFFLOAD_V1: '1'
+ NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: '128'
+ NUM_OF_TOKENS_PER_CHUNK_DISPATCH_API: '128'
+ HF_HUB_OFFLINE: '1'
+ TRANSFORMERS_OFFLINE: '1'
+ PYTHONWARNINGS: ignore
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: Qwen/Qwen3-235B-A22B
+ num_layers: 94
+ hidden_size: 4096
+ ffn_hidden_size: 12288
+ num_attention_heads: 64
+ kv_channels: 128
+ max_position_embeddings: 4096
+ group_query_attention: true
+ num_query_groups: 4
+ qk_layernorm: true
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_percent: 1.0
+ rotary_base: 1000000
+ make_vocab_size_divisible_by: 1187
+ num_experts: 128
+ moe_ffn_hidden_size: 1536
+ moe_router_load_balancing_type: aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: false
+ moe_aux_loss_coeff: 1e-3
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 1
+ pipeline_model_parallel_size: 1
+ expert_model_parallel_size: 64
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ account_for_embedding_in_pipeline_split: true
+ account_for_loss_in_pipeline_split: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: hybridep
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ moe_router_padding_for_quantization: true
+ moe_hybridep_num_sms: 32
+ cuda_graph_impl: transformer_engine
+ cuda_graph_scope:
+ - attn
+ - moe_router
+ - moe_preprocess
+ cuda_graph_warmup_steps: 2
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ te_rng_tracker: true
+ cross_entropy_fusion_impl: te
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 8192
+ train_samples: 268554688
+ exit_duration_in_mins: 230
+ distributed_timeout_minutes: 220
+ no_create_attention_mask_in_dataloader: true
+ cross_entropy_loss_fusion: true
+ manual_gc: true
+ manual_gc_interval: 5
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ fp8_format: e4m3
+ fp8_recipe: mxfp8
+ fp8_param_gather: true
+ reuse_grad_buf_for_mxfp8_param_ag: true
+ overlap_grad_reduce: true
+ overlap_param_gather: true
+ use_precision_aware_optimizer: true
+ main_grads_dtype: fp32
+ main_params_dtype: fp32
+ exp_avg_dtype: bf16
+ exp_avg_sq_dtype: bf16
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 500
+ finetune: true
+ auto_detect_ckpt_format: true
+ no_load_rng: true
+ no_load_optim: true
+ load: ${LOAD_PATH}
+ save: ${OUTPUT_PATH}/checkpoints
+ save_interval: 100
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_num_zeros_in_grad: true
+ log_params_norm: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_project: ${WANDB_PROJECT}
+ wandb_exp_name: Qwen3-235B-GB200-MXFP8-PartialCG-Overlap-TP1PP1EP64-MBS1GBS8192
+ enable_experimental: true
diff --git a/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml b/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml
new file mode 100644
index 00000000000..2b2d42e4494
--- /dev/null
+++ b/examples/moe_recipes/qwen3_235b/gb300/mxfp8_128GPU_TP1PP1EP64_paged_stash_full_cg.yaml
@@ -0,0 +1,247 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.04-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: gb200-gb300-2604-te215
+ #
+ # Variant of gb200-gb300-2604 with TransformerEngine main @ v2.15 and
+ # cuDNN-frontend bumped to >=1.23.0. Otherwise identical to
+ # Dockerfile.GB200.GB300.torch2604.
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ # TE v2.15 main
+ ARG TE_COMMIT="6cdd7115e1c1e6605feaff6ebe111ea0466cd71a"
+ # 100a = SM100a (GB200), 103a = SM103a (GB300)
+ ARG NVTE_CUDA_ARCHS="100a;103a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install latest cuDNN (libcudnn9-cuda-13) from NVIDIA's CUDA apt repo.
+ # cuda-keyring is arch-independent; repo path selects sbsa on aarch64 and x86_64 on amd64.
+ RUN bash -ex <<"EOF"
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") REPO_ARCH=x86_64 ;;
+ "aarch64") REPO_ARCH=sbsa ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ apt-get update
+ wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${REPO_ARCH}/cuda-keyring_1.1-1_all.deb
+ dpkg -i cuda-keyring_1.1-1_all.deb
+ rm cuda-keyring_1.1-1_all.deb
+ apt-get update
+ apt-get install -y --no-install-recommends libcudnn9-cuda-13
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}"
+
+ # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support
+ # NOTE: nvidia-cudnn-frontend>=1.23.0 (TE v2.15 CuTe DSL fused grouped MLP /
+ # grouped_gemm_glu_wrapper_sm100). Pin cutlass-dsl>=4.4.2 for cutlass module compatibility.
+ RUN pip install --no-cache-dir flash-attn-4==4.0.0b10 "nvidia-cutlass-dsl[cu13]>=4.4.2" && \
+ pip install --no-cache-dir "nvidia-cudnn-frontend[cutedsl]>=1.23.0"
+
+ # Install FlashMLA (DeepSeek MLA kernels; SM90 path disabled for Blackwell-only builds)
+ RUN FLASH_MLA_DISABLE_SM90=1 NVCC_THREADS=16 \
+ CFLAGS="-I${CUDA_HOME}/include/cccl" CXXFLAGS="-I${CUDA_HOME}/include/cccl" \
+ pip install --no-cache-dir --no-build-isolation \
+ git+https://github.com/deepseek-ai/FlashMLA.git
+
+ # =========================
+ # Option 1: HybridEP
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ # HybridEP latest hybrid-ep tip (DOCA path; older 1b8f4679 forced NIXL via .use_nixl marker)
+ cd DeepEP && git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6
+ rm -f .use_nixl
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 USE_NIXL=0 \
+ TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: graph_capture_record_stream_reuse:True
+ TORCH_NCCL_AVOID_RECORD_STREAMS: '0'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NCCL_NVLS_ENABLE: '0'
+ NCCL_GRAPH_REGISTER: '0'
+ NCCL_DEBUG: VERSION
+ NVTE_FUSED_ATTN: '1'
+ USE_MNNVL: '1'
+ NVTE_NORM_FWD_USE_CUDNN: '0'
+ NVTE_NORM_BWD_USE_CUDNN: '0'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '20'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '20'
+ NVTE_CPU_OFFLOAD_V1: '1'
+ PYTHONWARNINGS: ignore
+ NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: Qwen/Qwen3-235B-A22B
+ num_layers: 94
+ hidden_size: 4096
+ ffn_hidden_size: 12288
+ num_attention_heads: 64
+ kv_channels: 128
+ max_position_embeddings: 4096
+ group_query_attention: true
+ num_query_groups: 4
+ qk_layernorm: true
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_percent: 1.0
+ rotary_base: 1000000
+ make_vocab_size_divisible_by: 1187
+ num_experts: 128
+ moe_ffn_hidden_size: 1536
+ moe_router_load_balancing_type: aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: false
+ moe_aux_loss_coeff: 1e-3
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 1
+ pipeline_model_parallel_size: 1
+ expert_model_parallel_size: 64
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ account_for_embedding_in_pipeline_split: true
+ account_for_loss_in_pipeline_split: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: hybridep
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ moe_router_padding_for_quantization: true
+ moe_hybridep_num_sms: 32
+ moe_paged_stash: true
+ moe_expert_rank_capacity_factor: 1.2
+ moe_paged_stash_page_size: 64
+ moe_paged_stash_buffer_size_factor_cuda: 1.0
+ use_transformer_engine_op_fuser: true
+ moe_pad_experts_for_cuda_graph_inference: true
+ moe_mlp_glu_interleave_size: 32
+ cuda_graph_impl: local
+ cuda_graph_scope: full_iteration
+ cuda_graph_warmup_steps: 2
+ no_check_for_nan_in_loss_and_grad: true
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ te_rng_tracker: true
+ cross_entropy_fusion_impl: te
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 8192
+ train_iters: 20
+ distributed_timeout_minutes: 220
+ no_create_attention_mask_in_dataloader: true
+ cross_entropy_loss_fusion: true
+ manual_gc: true
+ manual_gc_interval: 5
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ fp8_format: e4m3
+ fp8_recipe: mxfp8
+ fp8_param_gather: true
+ reuse_grad_buf_for_mxfp8_param_ag: true
+ overlap_grad_reduce: true
+ overlap_param_gather: true
+ use_precision_aware_optimizer: true
+ main_grads_dtype: fp32
+ main_params_dtype: fp32
+ exp_avg_dtype: bf16
+ exp_avg_sq_dtype: bf16
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 500
+ finetune: true
+ auto_detect_ckpt_format: true
+ no_load_rng: true
+ no_load_optim: true
+ load: ${LOAD_PATH}
+ save: ${OUTPUT_PATH}/checkpoints
+ save_interval: 100
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_num_zeros_in_grad: true
+ log_params_norm: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_project: ${WANDB_PROJECT}
+ wandb_exp_name: Qwen3-235B-GB300-MXFP8-PagedStash-FullCG-TP1PP1EP64-MBS1GBS8192
+ enable_experimental: true
diff --git a/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml b/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml
new file mode 100644
index 00000000000..d2ebd988f8d
--- /dev/null
+++ b/examples/moe_recipes/qwen3_235b/h100/bf16_256GPU_TP2PP8EP32.yaml
@@ -0,0 +1,186 @@
+DEPENDENCIES:
+ pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3
+ dockerfile: |
+ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ # IMAGE_NAME: h100-torch2603
+
+ ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3
+ FROM ${FROM_IMAGE_NAME} AS base
+
+ ENV SHELL=/bin/bash
+ ENV DEBIAN_FRONTEND=noninteractive
+
+ ARG YQ_VERSION=4.27.5
+ ARG TE_TAG=release_v2.14
+ ARG NVTE_CUDA_ARCHS="90a"
+
+ # Install system dependencies
+ RUN bash -ex <<"EOF"
+ rm -rf /opt/megatron-lm
+ apt-get update
+ apt-get install -y --no-install-recommends git curl gettext sudo
+ ARCH=$(uname -m)
+ case "${ARCH}" in
+ "x86_64") YQ_ARCH=amd64 ;;
+ "aarch64") YQ_ARCH=arm64 ;;
+ *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;;
+ esac
+ wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq
+ chmod +x /usr/bin/yq
+ apt-get clean
+ rm -rf /var/lib/apt/lists/*
+ EOF
+
+ # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml)
+ RUN bash -ex <<"EOF"
+ unset PIP_CONSTRAINT
+ pip install \
+ sentencepiece tiktoken transformers accelerate \
+ wandb einops tqdm datasets nvtx \
+ flask-restful flask[async] fastapi hypercorn \
+ nltk wrapt pydantic pyyaml omegaconf \
+ pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \
+ coverage tensorboard
+ EOF
+
+ # Install TransformerEngine
+ RUN unset PIP_CONSTRAINT && \
+ NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \
+ pip install --no-build-isolation --no-cache-dir \
+ "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}"
+
+ # Install Flash Attention (standard, not FA4 which requires SM100+)
+ RUN pip install --no-cache-dir flash-attn
+
+ # =========================
+ # Option 1: HybridEP
+ # =========================
+ FROM base AS hybridep
+
+ RUN bash -ex <<"EOF"
+ cd /workspace
+ git clone https://github.com/linux-rdma/rdma-core.git
+ cd rdma-core && git checkout tags/v60.0 && sh build.sh
+ apt-get update
+ apt-get install -y --no-install-recommends libnvidia-ml-dev
+ git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git
+ cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316
+ RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \
+ TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \
+ pip install --no-build-isolation .
+ apt-get purge -y libnvidia-ml-dev
+ apt-get autoremove -y
+ rm -rf /root/.cache /tmp/* /var/lib/apt/lists/*
+ EOF
+
+ WORKDIR /workspace/
+ENV_VARS:
+ NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1'
+ PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+ NCCL_NVLS_ENABLE: '0'
+ NVTE_FUSED_ATTN: '1'
+ NVTE_NORM_FWD_USE_CUDNN: '1'
+ NVTE_NORM_BWD_USE_CUDNN: '1'
+ CUDA_DEVICE_MAX_CONNECTIONS: '32'
+ NVTE_FWD_LAYERNORM_SM_MARGIN: '24'
+ NVTE_BWD_LAYERNORM_SM_MARGIN: '24'
+ NCCL_GRAPH_REGISTER: '0'
+ARGS:
+ tokenizer_type: HuggingFaceTokenizer
+ tokenizer_model: Qwen/Qwen3-235B-A22B
+ num_layers: 94
+ hidden_size: 4096
+ ffn_hidden_size: 12288
+ num_attention_heads: 64
+ kv_channels: 128
+ max_position_embeddings: 4096
+ group_query_attention: true
+ num_query_groups: 4
+ qk_layernorm: true
+ normalization: RMSNorm
+ norm_epsilon: 1e-6
+ swiglu: true
+ disable_bias_linear: true
+ untie_embeddings_and_output_weights: true
+ position_embedding_type: rope
+ rotary_percent: 1.0
+ rotary_base: 1000000
+ make_vocab_size_divisible_by: 1187
+ num_experts: 128
+ moe_ffn_hidden_size: 1536
+ moe_router_load_balancing_type: aux_loss
+ moe_router_topk: 8
+ moe_router_pre_softmax: false
+ moe_aux_loss_coeff: 1e-3
+ attention_dropout: 0.0
+ hidden_dropout: 0.0
+ mock_data: true
+ seq_length: 4096
+ moe_router_force_load_balancing: true
+ tensor_model_parallel_size: 2
+ pipeline_model_parallel_size: 8
+ expert_model_parallel_size: 32
+ context_parallel_size: 1
+ expert_tensor_parallel_size: 1
+ num_layers_per_virtual_pipeline_stage: 3
+ use_distributed_optimizer: true
+ sequence_parallel: true
+ account_for_embedding_in_pipeline_split: true
+ account_for_loss_in_pipeline_split: true
+ moe_token_dispatcher_type: flex
+ moe_flex_dispatcher_backend: hybridep
+ moe_grouped_gemm: true
+ moe_permute_fusion: true
+ moe_router_fusion: true
+ moe_router_dtype: fp32
+ delay_wgrad_compute: true
+ overlap_moe_expert_parallel_comm: true
+ recompute_granularity: selective
+ recompute_modules: moe_act layernorm
+ cuda_graph_impl: transformer_engine
+ cuda_graph_scope: moe_router moe_preprocess
+ distributed_timeout_minutes: 220
+ use_mcore_models: true
+ use_flash_attn: true
+ transformer_impl: transformer_engine
+ micro_batch_size: 1
+ global_batch_size: 2048
+ train_samples: 268554688
+ exit_duration_in_mins: 230
+ no_create_attention_mask_in_dataloader: true
+ cross_entropy_loss_fusion: true
+ cross_entropy_fusion_impl: te
+ manual_gc: true
+ manual_gc_interval: 5
+ lr: 3.9e-06
+ min_lr: 3.9e-07
+ lr_warmup_init: 3.9e-07
+ lr_decay_style: cosine
+ lr_decay_samples: 584765624
+ lr_warmup_samples: 1536000
+ weight_decay: 0.1
+ clip_grad: 1.0
+ adam_beta1: 0.9
+ adam_beta2: 0.95
+ bf16: true
+ init_method_std: 0.02
+ eval_iters: 32
+ eval_interval: 500
+ finetune: true
+ auto_detect_ckpt_format: true
+ no_load_rng: true
+ no_load_optim: true
+ load: ${LOAD_PATH}
+ save_interval: 100
+ dist_ckpt_strictness: log_all
+ log_throughput: true
+ log_interval: 1
+ log_timers_to_tensorboard: true
+ log_memory_to_tensorboard: true
+ log_num_zeros_in_grad: true
+ log_params_norm: true
+ log_validation_ppl_to_tensorboard: true
+ logging_level: 40
+ tensorboard_dir: ${OUTPUT_PATH}/tensorboard
+ wandb_exp_name: Qwen3-235B-TP2PP8EP32-GBS2048
+ enable_experimental: true