diff --git a/examples/mimo/training/args.py b/examples/mimo/training/args.py new file mode 100644 index 00000000000..e1d9e2116f7 --- /dev/null +++ b/examples/mimo/training/args.py @@ -0,0 +1,145 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Hetero grid/topology CLI args + validation for the MIMO example.""" + +from __future__ import annotations + +import argparse +from typing import List + +from examples.mimo.training.topology import ModuleGridSpec +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY + + +def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """Register hetero parallelism args for the single-encoder MIMO example.""" + grid = parser.add_argument_group("hetero module grids") + + # Single encoder grid; CP/PP stay fixed at 1. + grid.add_argument("--encoder-tp", type=int, default=2, + help="Encoder tensor-model-parallel size.") + grid.add_argument("--encoder-dp", type=int, default=2, + help="Encoder data-parallel size.") + + # Language grid placement + factorization. + grid.add_argument("--llm-offset", type=int, default=4, + help="First global rank of the language grid span.") + grid.add_argument("--llm-tp", type=int, default=2, + help="Language tensor-model-parallel size.") + grid.add_argument("--llm-cp", type=int, default=1, + help="Language context-parallel size (CP=1 only for now).") + grid.add_argument("--llm-pp", type=int, default=1, + help="Language pipeline-model-parallel size.") + grid.add_argument("--llm-dp", type=int, default=2, + help="Language data-parallel size. Global batch is keyed on this.") + # MoE expert parallelism for the language grid. + grid.add_argument("--llm-ep", type=int, default=1, + help="Language expert-model-parallel size (MoE).") + grid.add_argument("--llm-expt-tp", type=int, default=None, + help="Language expert tensor-parallel size; defaults to 1 when unset " + "(experts default to TP=1; the 20L MoE recipe passes --llm-expt-tp 1).") + + grid.add_argument( + "--llm-only", + action="store_true", + help=( + "Run only the MIMO language module on the LLM grid. Keeps the MIMO " + "training/data path but creates no encoder ranks or bridge communicators; " + "requires --llm-offset 0 so the language grid covers WORLD_SIZE." + ), + ) + return parser + + +def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: + """Validate the disjoint hetero grid layout; returns ``(encoder_size, llm_size)``.""" + if args.llm_cp != 1: + raise ValueError("hetero MIMO training currently supports CP=1 only") + + # MoE expert count must divide evenly across the language grid's expert parallelism. + num_experts = _num_experts(args) + if num_experts and num_experts % args.llm_ep != 0: + raise ValueError( + f"--num-experts ({num_experts}) must be divisible by --llm-ep ({args.llm_ep})" + ) + + llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp + + if args.llm_only: + if args.llm_offset != 0: + raise ValueError( + "--llm-only requires --llm-offset 0 so language ranks cover WORLD_SIZE" + ) + llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) + all_ranks = set(range(world_size)) + if llm_ranks != all_ranks: + raise ValueError( + "--llm-only requires the language grid to cover every torchrun rank exactly " + f"once; covered={sorted(llm_ranks)}, world={sorted(all_ranks)}" + ) + return 0, llm_size + + # Fan-out divisibility: the bridge splits (mbs * llm_dp) LLM lanes across + # encoder_dp encoder lanes; the split must be exact. + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError( + "--micro-batch-size * --llm-dp must be divisible by --encoder-dp " + f"(got {args.micro_batch_size} * {args.llm_dp} % {args.encoder_dp} != 0)" + ) + + encoder_size = args.encoder_tp * args.encoder_dp + encoder_ranks = set(range(encoder_size)) # encoder span always starts at rank 0 + llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) + all_ranks = set(range(world_size)) + + if not encoder_ranks.isdisjoint(llm_ranks): + raise ValueError( + "hetero MIMO expects disjoint module rank spans; " + f"spans overlap at {sorted(encoder_ranks & llm_ranks)}" + ) + if encoder_ranks | llm_ranks != all_ranks: + raise ValueError( + "The non-colocated module grids must cover every torchrun rank exactly once; " + f"covered={sorted(encoder_ranks | llm_ranks)}, world={sorted(all_ranks)}" + ) + + return encoder_size, llm_size + + +def build_module_grid_specs( + args: argparse.Namespace, world_size: int, encoder_module_name: str +) -> List[ModuleGridSpec]: + """Map grid args to the ModuleGridSpec list create_topology consumes.""" + encoder_size, llm_size = validate_hetero_grid_args(args, world_size) + + language_grid_spec = ModuleGridSpec( + name=MIMO_LANGUAGE_MODULE_KEY, + num_ranks=llm_size, + tp=args.llm_tp, + cp=args.llm_cp, + pp=args.llm_pp, + ep=args.llm_ep, + rank_offset=args.llm_offset, + expt_tp=args.llm_expt_tp or 1, + ) + + if args.llm_only: + return [language_grid_spec] + + encoder_grid_spec = ModuleGridSpec( + name=encoder_module_name, + num_ranks=encoder_size, + tp=args.encoder_tp, + cp=1, + pp=1, + ep=1, + rank_offset=0, + expt_tp=1, + ) + return [encoder_grid_spec, language_grid_spec] + + +def _num_experts(args: argparse.Namespace) -> int: + """Resolve MoE expert count from the stock --num-experts arg.""" + value = getattr(args, "num_experts", None) + return int(value) if value else 0 diff --git a/examples/mimo/training/topology.py b/examples/mimo/training/topology.py index cf22de86627..b3fa4c94d14 100644 --- a/examples/mimo/training/topology.py +++ b/examples/mimo/training/topology.py @@ -132,7 +132,11 @@ def _build_grid(spec: ModuleGridSpec) -> HyperCommGrid: ) try: - for dims in (["tp"], ["cp"], ["pp"], ["dp"], ["dp", "cp"], ["tp", "cp"], ["tp", "pp"]): + for dims in ( + ["tp"], ["cp"], ["pp"], ["dp"], + ["dp", "cp"], ["tp", "cp"], ["tp", "pp"], + ["tp", "dp"], ["tp", "dp", "cp"], ["tp", "cp", "dp", "pp"], + ): grid.create_pg(dims) for dims in (["ep"], ["expt_tp"], ["expt_dp"], ["expt_tp", "ep"], ["expt_tp", "ep", "pp"]): grid.create_pg(dims, view=_EXPERT_VIEW) @@ -191,6 +195,8 @@ def pg_collection_from_grid( pgc.dp_cp = grid.get_pg(["dp", "cp"]) pgc.intra_dp_cp = pgc.dp_cp pgc.tp_cp = grid.get_pg(["tp", "cp"]) + pgc.tp_dp = grid.get_pg(["tp", "dp"]) + pgc.tp_dp_cp = grid.get_pg(["tp", "dp", "cp"]) pgc.mp = grid.get_pg(["tp", "pp"]) pgc.ep = grid.get_pg("ep", view=_EXPERT_VIEW) pgc.expt_tp = grid.get_pg("expt_tp", view=_EXPERT_VIEW) diff --git a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py new file mode 100644 index 00000000000..7429e87434e --- /dev/null +++ b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py @@ -0,0 +1,119 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Pure-args (no-GPU) tests for the hetero grid arg group + validation.""" + +from __future__ import annotations + +import argparse + +import pytest + +from examples.mimo.training.args import ( + add_hetero_grid_args, + build_module_grid_specs, + validate_hetero_grid_args, +) +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY + +WORLD_SIZE_8 = 8 + + +def _parse(argv): + """Parse only the hetero grid args from a token list.""" + parser = argparse.ArgumentParser() + add_hetero_grid_args(parser) + return parser.parse_args(argv) + + +def _layout_8gpu_20l(**overrides): + """Canonical 8-GPU layout: encoder 0-3 (tp2/dp2), llm 4-7 (tp2/pp1/dp2/ep4).""" + argv = ( + "--encoder-tp 2 --encoder-dp 2 " + "--llm-offset 4 --llm-tp 2 --llm-pp 1 --llm-dp 2 --llm-ep 4" + ).split() + args = _parse(argv) + # Stock args the validator reads but the grid parser does not own. + args.micro_batch_size = 1 + args.num_experts = 128 + for key, value in overrides.items(): + setattr(args, key, value) + return args + + +def test_canonical_layout_validates_and_maps_specs(): + args = _layout_8gpu_20l() + encoder_size, llm_size = validate_hetero_grid_args(args, WORLD_SIZE_8) + assert (encoder_size, llm_size) == (4, 4) + + encoder_grid_spec, language_grid_spec = build_module_grid_specs( + args, WORLD_SIZE_8, encoder_module_name="radio_encoder" + ) + assert encoder_grid_spec.name == "radio_encoder" + assert encoder_grid_spec.num_ranks == 4 + assert encoder_grid_spec.rank_offset == 0 # encoder span always starts at rank 0 + assert encoder_grid_spec.cp == 1 + assert encoder_grid_spec.pp == 1 + assert encoder_grid_spec.dp == 2 # derived: 4 // tp2 + assert language_grid_spec.name == MIMO_LANGUAGE_MODULE_KEY + assert language_grid_spec.num_ranks == 4 + assert language_grid_spec.rank_offset == 4 + assert language_grid_spec.dp == 2 + # expt_tp defaults to 1 when --llm-expt-tp unset (ep=4 over 4 ranks needs expt_tp=1). + assert language_grid_spec.expt_tp == 1 + + +def test_overlapping_spans_raise(): + # llm-offset 2 makes llm ranks {2,3,4,5} overlap encoder ranks {0,1,2,3}. + args = _layout_8gpu_20l(llm_offset=2) + with pytest.raises(ValueError, match="disjoint"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_non_covering_spans_raise(): + # encoder 0-3 + llm 4-7 cover only 8 ranks; declare world_size 10 -> gap. + args = _layout_8gpu_20l() + with pytest.raises(ValueError, match="cover every torchrun rank"): + validate_hetero_grid_args(args, 10) + + +def test_fanout_divisibility_raises(): + # mbs(1) * llm_dp(2) = 2 not divisible by encoder_dp(3). + args = _layout_8gpu_20l(encoder_dp=3, micro_batch_size=1, llm_dp=2) + with pytest.raises(ValueError, match="divisible by --encoder-dp"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_ep_divisibility_raises(): + # num_experts 128 not divisible by llm_ep 3. + args = _layout_8gpu_20l(llm_ep=3, num_experts=128) + with pytest.raises(ValueError, match="divisible by --llm-ep"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_parser_does_not_expose_unsupported_grid_knobs(): + args = _parse([]) + assert not hasattr(args, "encoder_cp") + assert not hasattr(args, "encoder_pp") + assert not hasattr(args, "llm_expt_dp") + + +def test_llm_cp_must_be_one(): + args = _layout_8gpu_20l(llm_cp=2) + with pytest.raises(ValueError, match="CP=1 only"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_llm_only_requires_offset_zero(): + args = _layout_8gpu_20l(llm_only=True, llm_offset=4) + with pytest.raises(ValueError, match="--llm-only requires --llm-offset 0"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_llm_only_covers_world(): + # llm tp2/pp1/dp2 = 4 ranks at offset 0; world_size 4 -> covers exactly, no encoder spec. + args = _layout_8gpu_20l(llm_only=True, llm_offset=0, llm_ep=2, num_experts=128) + encoder_size, llm_size = validate_hetero_grid_args(args, 4) + assert (encoder_size, llm_size) == (0, 4) + specs = build_module_grid_specs(args, 4, encoder_module_name="radio_encoder") + assert len(specs) == 1 + assert specs[0].name == MIMO_LANGUAGE_MODULE_KEY