perf: enable HybridEP in H100/B200 performance recipes

examples/configs/recipes/llm/performance/dapo-deepseek-v3-64n8g.v2.yaml

@@ -41,6 +41,15 @@ policy:
   make_sequence_length_divisible_by: ${mul:${policy.megatron_cfg.tensor_model_parallel_size},
     ${mul:2, ${policy.megatron_cfg.context_parallel_size}}}
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     empty_unused_memory_level: 2
     enabled: true
     activation_checkpointing: true
@@ -99,4 +108,3 @@ logger:
 cluster:
   gpus_per_node: 8
   num_nodes: 64
-

examples/configs/recipes/llm/performance/grpo-deepseek-v3-32n4g.yaml

@@ -10,6 +10,11 @@ policy:
     moe_token_dispatcher_type: flex
     moe_flex_dispatcher_backend: hybridep
     moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "16"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "72"
+      USE_MNNVL: "1"
   generation:
     vllm_cfg:
       gpu_memory_utilization: 0.3

examples/configs/recipes/llm/performance/grpo-deepseek-v3-32n8g.yaml

@@ -21,6 +21,15 @@ policy:
   dtensor_cfg:
     enabled: false
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     enabled: true
     empty_unused_memory_level: 1
     pipeline_model_parallel_size: 16

examples/configs/recipes/llm/performance/grpo-deepseek-v3-64n4g-async-1off.yaml

@@ -3,6 +3,15 @@ checkpointing:
   checkpoint_dir: results/grpo-deepseek-v3-64n4g-async-1off
 policy:
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "16"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "72"
+      USE_MNNVL: "1"
     pipeline_model_parallel_size: 8
     num_layers_in_first_pipeline_stage: 7
     num_layers_in_last_pipeline_stage: 6

examples/configs/recipes/llm/performance/grpo-deepseek-v3-64n4g.yaml

@@ -8,5 +8,7 @@ logger:
 policy:
   megatron_cfg:
     expert_model_parallel_size: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "32"
 cluster:
   num_nodes: 64

examples/configs/recipes/llm/performance/grpo-deepseek-v3-64n8g-async-1off.yaml

@@ -11,6 +11,15 @@ checkpointing:
 policy:
   logprob_batch_size: 2
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     tensor_model_parallel_size: 1
     pipeline_model_parallel_size: 16
     expert_model_parallel_size: 16

examples/configs/recipes/llm/performance/grpo-deepseek-v3-64n8g-fp8-async-1off.yaml

@@ -3,6 +3,10 @@ checkpointing:
   checkpoint_dir: results/grpo-deepseek-v3-64n8g-fp8-async-1off
 policy:
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
     fp8_cfg:
       enabled: true
       fp8: "e4m3"
@@ -11,6 +15,10 @@ policy:
     moe_router_dtype: fp32
     env_vars:
       NVTE_FP8_BLOCK_SCALING_FP32_SCALES: "1"
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
   generation:
     vllm_cfg:
       tensor_parallel_size: 16

examples/configs/recipes/llm/performance/grpo-deepseek-v3-64n8g.yaml

@@ -7,6 +7,15 @@ logger:
     name: grpo-deepseek-v3-64n8g
 policy:
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     expert_model_parallel_size: 32
 cluster:
   num_nodes: 64

examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n8g-async-1off.yaml

@@ -25,6 +25,16 @@ policy:
   dtensor_cfg:
     enabled: false
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    moe_hybridep_prepad_packed_inputs: true
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     enabled: true
     tensor_model_parallel_size: 4
     expert_model_parallel_size: 32
@@ -59,4 +69,4 @@ logger:
     name: grpo-nemotron3-super-120BA12B-32n8g-async-1off
 cluster:
   gpus_per_node: 8
-  num_nodes: 32
+  num_nodes: 32

examples/configs/recipes/llm/performance/grpo-nemotron3-super-120BA12B-32n8g.yaml

@@ -20,6 +20,16 @@ policy:
   dtensor_cfg:
     enabled: false
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    moe_hybridep_prepad_packed_inputs: true
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     enabled: true
     tensor_model_parallel_size: 4
     expert_model_parallel_size: 32
@@ -49,4 +59,4 @@ logger:
     name: grpo-nemotron3-super-120BA12B-32n8g
 cluster:
   gpus_per_node: 8
-  num_nodes: 32
+  num_nodes: 32

examples/configs/recipes/llm/performance/grpo-qwen3-235b-16n4g.yaml

@@ -10,6 +10,12 @@ policy:
     moe_token_dispatcher_type: flex
     moe_flex_dispatcher_backend: hybridep
     moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "16"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "72"
+      USE_MNNVL: "1"
+      PYTORCH_CUDA_ALLOC_CONF: expandable_segments:False
   generation:
     vllm_cfg:
       tensor_parallel_size: 8

examples/configs/recipes/llm/performance/grpo-qwen3-235b-16n8g.yaml

@@ -23,6 +23,15 @@ policy:
   dtensor_cfg:
     enabled: false
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     enabled: true
     empty_unused_memory_level: 1
     tensor_model_parallel_size: 2

examples/configs/recipes/llm/performance/grpo-qwen3-235b-32n4g-async-1off.yaml

@@ -3,6 +3,15 @@ checkpointing:
   checkpoint_dir: results/grpo-qwen3-235b-32n4g-async-1off
 policy:
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "16"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "72"
+      USE_MNNVL: "1"
     pipeline_model_parallel_size: 4
     num_layers_in_first_pipeline_stage: 23
     num_layers_in_last_pipeline_stage: 23

examples/configs/recipes/llm/performance/grpo-qwen3-235b-32n8g-async-1off.yaml

@@ -10,6 +10,15 @@ checkpointing:
   checkpoint_dir: results/grpo-qwen3-235b-32n8g-async-1off
 policy:
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     tensor_model_parallel_size: 4
     sequence_parallel: true
     context_parallel_size: 1

examples/configs/recipes/llm/performance/grpo-qwen3-235b-32n8g.yaml

@@ -5,5 +5,16 @@ logger:
   log_dir: logs/grpo-qwen3-235b-32n8g
   wandb:
     name: grpo-qwen3-235b-32n8g
+policy:
+  megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
 cluster:
   num_nodes: 32

examples/configs/recipes/llm/performance/grpo-qwen3-30ba3b-24n8g-async-8off.yaml

@@ -12,6 +12,16 @@ checkpointing:
 policy:
   train_global_batch_size: 512
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    moe_hybridep_prepad_packed_inputs: true
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     tensor_model_parallel_size: 1
     pipeline_model_parallel_size: 1
     expert_model_parallel_size: 8

examples/configs/recipes/llm/performance/grpo-qwen3-30ba3b-4n8g-40K.yaml

@@ -19,6 +19,11 @@ policy:
   make_sequence_length_divisible_by: ${mul:${policy.megatron_cfg.tensor_model_parallel_size},
     ${mul:2, ${policy.megatron_cfg.context_parallel_size}}}
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    moe_hybridep_prepad_packed_inputs: true
     enabled: true
     empty_unused_memory_level: 1
     tensor_model_parallel_size: 4
@@ -35,6 +40,10 @@ policy:
       lr_warmup_init: 3.0e-08
     env_vars:
       PYTORCH_CUDA_ALLOC_CONF: expandable_segments:False
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     activation_checkpointing: true
   generation:
     vllm_cfg:

examples/configs/recipes/llm/performance/grpo-qwen3-30ba3b-4n8g-async-1off.yaml

@@ -12,6 +12,16 @@ checkpointing:
 policy:
   train_global_batch_size: 2048
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    moe_hybridep_prepad_packed_inputs: false
+    env_vars:
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
     tensor_model_parallel_size: 1
     pipeline_model_parallel_size: 2
     expert_model_parallel_size: 8

examples/configs/recipes/llm/performance/grpo-qwen3-30ba3b-4n8g.yaml

@@ -12,6 +12,7 @@ policy:
   train_global_batch_size: 2048
   train_micro_batch_size: 1
   logprob_batch_size: 2
+  logprob_chunk_size: 1024
   max_total_sequence_length: 4096
   sequence_packing:
     fuse_loss: true
@@ -21,6 +22,12 @@ policy:
   scheduler: null
   make_sequence_length_divisible_by: ${policy.megatron_cfg.tensor_model_parallel_size}
   megatron_cfg:
+    # HybridEP settings
+    moe_token_dispatcher_type: flex
+    moe_flex_dispatcher_backend: hybridep
+    moe_hybridep_num_sms: 32
+    moe_hybridep_prepad_packed_inputs: true
+    defer_fp32_logits: true
     enabled: true
     empty_unused_memory_level: 1
     tensor_model_parallel_size: 1
@@ -36,6 +43,10 @@ policy:
       lr_warmup_init: 3.0e-08
     env_vars:
       PYTORCH_CUDA_ALLOC_CONF: expandable_segments:False
+      NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN: "8"
+      NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: "128"
+      NVLINK_DOMAIN_SIZE: "8"
+      USE_MNNVL: "0"
   generation:
     vllm_cfg:
       tensor_parallel_size: 2

experiment_logs/hybridep-performance-recipes-20260808/submit_recipe_tests.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -euo pipefail
+
+mode=${1:-submit}
+case "${mode}" in
+  submit) submit_mode=(--parsable) ;;
+  test-only) submit_mode=(--test-only) ;;
+  *) printf 'Usage: %s [submit|test-only]\n' "$0" >&2; exit 2 ;;
+esac
+
+work_root=/lustre/fsw/portfolios/coreai/projects/coreai_chef_posttrain/users/sna
+experiment_root=${work_root}/experiments/hybridep-performance-recipes-20260808
+repo=${VALIDATION_REPO_OVERRIDE:?VALIDATION_REPO_OVERRIDE is required}
+container=${CONTAINER_OVERRIDE:-${work_root}/containers/nemo-rl-nightly-cw-fallback-20260808/nemo_rl_nightly_20260805_15171871.sqsh}
+validation_head=${VALIDATION_HEAD_OVERRIDE:?VALIDATION_HEAD_OVERRIDE is required}
+run_root=${experiment_root}/runs/recipe-tests-${validation_head:0:12}
+job_reaper_comment='{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"15","reason":"other","description":"Focused NeMo-RL HybridEP recipe configuration tests"}}'
+
+test "$(git -C "${repo}" rev-parse HEAD)" = "${validation_head}"
+test -z "$(git -C "${repo}" status --porcelain --untracked-files=no)"
+test -r "${container}"
+mkdir -p "${run_root}/ray"
+
+COMMAND="PYTHONPATH=${repo}:\${PYTHONPATH:-} PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 /opt/nemo_rl_venv/bin/python -m pytest -q ${repo}/tests/unit/tools/test_hybridep_default_8g_recipes.py"
+export COMMAND
+export CONTAINER="${container}"
+export MOUNTS=/lustre:/lustre
+export BASE_LOG_DIR="${run_root}/ray"
+export GPUS_PER_NODE=8
+
+cd "${repo}"
+sbatch "${submit_mode[@]}" \
+  --export=ALL \
+  --nodes=1 \
+  --gpus-per-node=8 \
+  --exclusive \
+  --account=coreai_chef_posttrain \
+  --partition=batch \
+  --time=00:15:00 \
+  --job-name=coreai_chef_posttrain.hybridep-recipe-test \
+  --output="${run_root}/slurm-%j.out" \
+  --comment="${job_reaper_comment}" \
+  ray.sub