Repository navigation
Add hetero grid args and MoE process groups for MIMO example #5375
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
225f4ea
c3f466d
82e07c7
e1af6e3
e3a0b7c
4f0a1f9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. | ||
|
|
||
| """Hetero grid/topology CLI args + validation for the MIMO example.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| from typing import List | ||
|
|
||
| from examples.mimo.training.topology import ModuleGridSpec | ||
| from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY | ||
|
|
||
|
|
||
| def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: | ||
| """Register the hetero per-module parallelism args (--encoder-*/--llm-*).""" | ||
| grid = parser.add_argument_group("hetero module grids") | ||
|
|
||
| # Encoder grid factorization (the encoder span always starts at rank 0). | ||
| grid.add_argument("--encoder-tp", type=int, default=2, | ||
| help="Encoder tensor-model-parallel size.") | ||
| grid.add_argument("--encoder-cp", type=int, default=1, | ||
| help="Encoder context-parallel size (CP=1 only for now).") | ||
| grid.add_argument("--encoder-pp", type=int, default=1, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. so we now allow encoder to have PP and CP?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we could have PP, and CP we have pending PR. so removed these for now. will add back after full support exists |
||
| help="Encoder pipeline-model-parallel size.") | ||
| grid.add_argument("--encoder-dp", type=int, default=2, | ||
| help="Encoder data-parallel size.") | ||
|
|
||
| # Language grid placement + factorization. | ||
| grid.add_argument("--llm-offset", type=int, default=4, | ||
| help="First global rank of the language grid span.") | ||
| grid.add_argument("--llm-tp", type=int, default=2, | ||
|
yashaswikarnati marked this conversation as resolved.
Resolved
|
||
| help="Language tensor-model-parallel size.") | ||
| grid.add_argument("--llm-cp", type=int, default=1, | ||
| help="Language context-parallel size (CP=1 only for now).") | ||
| grid.add_argument("--llm-pp", type=int, default=1, | ||
| help="Language pipeline-model-parallel size.") | ||
| grid.add_argument("--llm-dp", type=int, default=2, | ||
| help="Language data-parallel size. Global batch is keyed on this.") | ||
| # MoE expert parallelism for the language grid. | ||
| grid.add_argument("--llm-ep", type=int, default=1, | ||
| help="Language expert-model-parallel size (MoE).") | ||
| grid.add_argument("--llm-expt-tp", type=int, default=None, | ||
| help="Language expert tensor-parallel size; defaults to 1 when unset " | ||
| "(experts default to TP=1; the 20L MoE recipe passes --llm-expt-tp 1).") | ||
| grid.add_argument("--llm-expt-dp", type=int, default=None, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. just for my understanding: any use case where user needs to provide expt-dp, instead of deriving it from the grid?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. good catch, we always derive expt dp based on world size and other paralleisms. removed this arg |
||
| help="Language expert data-parallel size; derived from the grid when unset.") | ||
|
|
||
| grid.add_argument( | ||
| "--llm-only", | ||
| action="store_true", | ||
| help=( | ||
| "Run only the MIMO language module on the LLM grid. Keeps the MIMO " | ||
| "training/data path but creates no encoder ranks or bridge communicators; " | ||
| "requires --llm-offset 0 so the language grid covers WORLD_SIZE." | ||
| ), | ||
| ) | ||
| return parser | ||
|
|
||
|
|
||
| def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: | ||
| """Validate the disjoint hetero grid layout; returns ``(encoder_size, llm_size)``.""" | ||
| if args.encoder_cp != 1 or args.llm_cp != 1: | ||
| raise ValueError("hetero MIMO training currently supports CP=1 only") | ||
|
|
||
| # MoE expert count must divide evenly across the language grid's expert parallelism. | ||
| num_experts = _num_experts(args) | ||
| if num_experts and num_experts % args.llm_ep != 0: | ||
| raise ValueError( | ||
| f"--num-experts ({num_experts}) must be divisible by --llm-ep ({args.llm_ep})" | ||
| ) | ||
|
|
||
| llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp | ||
|
|
||
| if args.llm_only: | ||
| if args.llm_offset != 0: | ||
| raise ValueError( | ||
| "--llm-only requires --llm-offset 0 so language ranks cover WORLD_SIZE" | ||
| ) | ||
| llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) | ||
| all_ranks = set(range(world_size)) | ||
| if llm_ranks != all_ranks: | ||
| raise ValueError( | ||
| "--llm-only requires the language grid to cover every torchrun rank exactly " | ||
| f"once; covered={sorted(llm_ranks)}, world={sorted(all_ranks)}" | ||
| ) | ||
| return 0, llm_size | ||
|
|
||
| # Fan-out divisibility: the bridge splits (mbs * llm_dp) LLM lanes across | ||
| # encoder_dp encoder lanes; the split must be exact. | ||
| if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: | ||
| raise ValueError( | ||
| "--micro-batch-size * --llm-dp must be divisible by --encoder-dp " | ||
| f"(got {args.micro_batch_size} * {args.llm_dp} % {args.encoder_dp} != 0)" | ||
| ) | ||
|
|
||
| encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp | ||
| encoder_ranks = set(range(encoder_size)) # encoder span always starts at rank 0 | ||
| llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) | ||
| all_ranks = set(range(world_size)) | ||
|
|
||
| if not encoder_ranks.isdisjoint(llm_ranks): | ||
| raise ValueError( | ||
| "hetero MIMO expects disjoint module rank spans; " | ||
| f"spans overlap at {sorted(encoder_ranks & llm_ranks)}" | ||
| ) | ||
| if encoder_ranks | llm_ranks != all_ranks: | ||
| raise ValueError( | ||
| "The non-colocated module grids must cover every torchrun rank exactly once; " | ||
| f"covered={sorted(encoder_ranks | llm_ranks)}, world={sorted(all_ranks)}" | ||
| ) | ||
|
|
||
| return encoder_size, llm_size | ||
|
|
||
|
|
||
| def build_module_grid_specs( | ||
| args: argparse.Namespace, world_size: int, encoder_module_name: str | ||
| ) -> List[ModuleGridSpec]: | ||
| """Map grid args to the ModuleGridSpec list create_topology consumes; caller supplies encoder_module_name.""" | ||
| encoder_size, llm_size = validate_hetero_grid_args(args, world_size) | ||
|
|
||
| language_grid_spec = ModuleGridSpec( | ||
| name=MIMO_LANGUAGE_MODULE_KEY, | ||
| num_ranks=llm_size, | ||
| tp=args.llm_tp, | ||
| cp=args.llm_cp, | ||
| pp=args.llm_pp, | ||
| ep=args.llm_ep, | ||
| rank_offset=args.llm_offset, | ||
| expt_tp=args.llm_expt_tp or 1, | ||
| ) | ||
|
|
||
| if args.llm_only: | ||
| return [language_grid_spec] | ||
|
|
||
| encoder_grid_spec = ModuleGridSpec( | ||
| name=encoder_module_name, | ||
| num_ranks=encoder_size, | ||
| tp=args.encoder_tp, | ||
| cp=args.encoder_cp, | ||
| pp=args.encoder_pp, | ||
| ep=1, | ||
| rank_offset=0, | ||
| expt_tp=1, | ||
| ) | ||
| return [encoder_grid_spec, language_grid_spec] | ||
|
|
||
|
|
||
| def _num_experts(args: argparse.Namespace) -> int: | ||
| """Resolve MoE expert count from the stock --num-experts arg.""" | ||
| value = getattr(args, "num_experts", None) | ||
| return int(value) if value else 0 | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. | ||
|
|
||
| """Pure-args (no-GPU) tests for the hetero grid arg group + validation.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
|
|
||
| import pytest | ||
|
|
||
| from examples.mimo.training.args import ( | ||
| add_hetero_grid_args, | ||
| build_module_grid_specs, | ||
| validate_hetero_grid_args, | ||
| ) | ||
| from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY | ||
|
|
||
| WORLD_SIZE_8 = 8 | ||
|
|
||
|
|
||
| def _parse(argv): | ||
| """Parse only the hetero grid args from a token list.""" | ||
| parser = argparse.ArgumentParser() | ||
| add_hetero_grid_args(parser) | ||
| return parser.parse_args(argv) | ||
|
|
||
|
|
||
| def _layout_8gpu_20l(**overrides): | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. tests not adding value, just smoke tests should be avoided. tests can be authored more concise here |
||
| """Canonical 8-GPU layout: encoder 0-3 (tp2/pp1/dp2), llm 4-7 (tp2/pp1/dp2/ep4).""" | ||
| argv = ( | ||
| "--encoder-tp 2 --encoder-pp 1 --encoder-dp 2 " | ||
| "--llm-offset 4 --llm-tp 2 --llm-pp 1 --llm-dp 2 --llm-ep 4" | ||
| ).split() | ||
| args = _parse(argv) | ||
| # Stock args the validator reads but the grid parser does not own. | ||
| args.micro_batch_size = 1 | ||
| args.num_experts = 128 | ||
| for key, value in overrides.items(): | ||
| setattr(args, key, value) | ||
| return args | ||
|
|
||
|
|
||
| def test_canonical_layout_validates_and_maps_specs(): | ||
| args = _layout_8gpu_20l() | ||
| encoder_size, llm_size = validate_hetero_grid_args(args, WORLD_SIZE_8) | ||
| assert (encoder_size, llm_size) == (4, 4) | ||
|
|
||
| encoder_grid_spec, language_grid_spec = build_module_grid_specs( | ||
| args, WORLD_SIZE_8, encoder_module_name="radio_encoder" | ||
| ) | ||
| assert encoder_grid_spec.name == "radio_encoder" | ||
| assert encoder_grid_spec.num_ranks == 4 | ||
| assert encoder_grid_spec.rank_offset == 0 # encoder span always starts at rank 0 | ||
| assert encoder_grid_spec.dp == 2 # derived: 4 // (tp2 * cp1 * pp1) | ||
| assert language_grid_spec.name == MIMO_LANGUAGE_MODULE_KEY | ||
| assert language_grid_spec.num_ranks == 4 | ||
| assert language_grid_spec.rank_offset == 4 | ||
| assert language_grid_spec.dp == 2 | ||
| # expt_tp defaults to 1 when --llm-expt-tp unset (ep=4 over 4 ranks needs expt_tp=1). | ||
| assert language_grid_spec.expt_tp == 1 | ||
|
|
||
|
|
||
| def test_overlapping_spans_raise(): | ||
| # llm-offset 2 makes llm ranks {2,3,4,5} overlap encoder ranks {0,1,2,3}. | ||
| args = _layout_8gpu_20l(llm_offset=2) | ||
| with pytest.raises(ValueError, match="disjoint"): | ||
| validate_hetero_grid_args(args, WORLD_SIZE_8) | ||
|
|
||
|
|
||
| def test_non_covering_spans_raise(): | ||
| # encoder 0-3 + llm 4-7 cover only 8 ranks; declare world_size 10 -> gap. | ||
| args = _layout_8gpu_20l() | ||
| with pytest.raises(ValueError, match="cover every torchrun rank"): | ||
| validate_hetero_grid_args(args, 10) | ||
|
|
||
|
|
||
| def test_fanout_divisibility_raises(): | ||
| # mbs(1) * llm_dp(2) = 2 not divisible by encoder_dp(3). | ||
| args = _layout_8gpu_20l(encoder_dp=3, micro_batch_size=1, llm_dp=2) | ||
| with pytest.raises(ValueError, match="divisible by --encoder-dp"): | ||
| validate_hetero_grid_args(args, WORLD_SIZE_8) | ||
|
|
||
|
|
||
| def test_ep_divisibility_raises(): | ||
| # num_experts 128 not divisible by llm_ep 3. | ||
| args = _layout_8gpu_20l(llm_ep=3, num_experts=128) | ||
| with pytest.raises(ValueError, match="divisible by --llm-ep"): | ||
| validate_hetero_grid_args(args, WORLD_SIZE_8) | ||
|
|
||
|
|
||
| def test_cp_must_be_one(): | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remove |
||
| args = _layout_8gpu_20l(llm_cp=2) | ||
| with pytest.raises(ValueError, match="CP=1 only"): | ||
| validate_hetero_grid_args(args, WORLD_SIZE_8) | ||
|
|
||
|
|
||
| def test_llm_only_requires_offset_zero(): | ||
|
yashaswikarnati marked this conversation as resolved.
Resolved
|
||
| args = _layout_8gpu_20l(llm_only=True, llm_offset=4) | ||
| with pytest.raises(ValueError, match="--llm-only requires --llm-offset 0"): | ||
| validate_hetero_grid_args(args, WORLD_SIZE_8) | ||
|
|
||
|
|
||
| def test_llm_only_covers_world(): | ||
| # llm tp2/pp1/dp2 = 4 ranks at offset 0; world_size 4 -> covers exactly, no encoder spec. | ||
| args = _layout_8gpu_20l(llm_only=True, llm_offset=0, llm_ep=2, num_experts=128) | ||
| encoder_size, llm_size = validate_hetero_grid_args(args, 4) | ||
| assert (encoder_size, llm_size) == (0, 4) | ||
| specs = build_module_grid_specs(args, 4, encoder_module_name="radio_encoder") | ||
| assert len(specs) == 1 | ||
| assert specs[0].name == MIMO_LANGUAGE_MODULE_KEY | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
how do we provide TP/PP/CP/etc if we have multiple encoder?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For this initial milestone of e2e training loop, targetting only single encoder and LLM. will revist this once we start pipe clearning on multiple encoders and LLM