From 225f4eade512c8c608c5b9f699590e0f98f9c4bf Mon Sep 17 00:00:00 2001 From: ykarnati Date: Tue, 16 Jun 2026 13:07:12 -0700 Subject: [PATCH 1/6] mimo example: add hetero grid/topology args and MoE process groups Add the heterogeneous-grid CLI args module (add_hetero_grid_args, validate_hetero_grid_args, build_module_grid_specs) used to configure per-module grids for MIMO training, and extend the example topology helper: - topology.py: create the additional dense process groups (tp+dp, tp+dp+cp, tp+cp+dp+pp) and set pgc.tp_dp / pgc.tp_dp_cp explicitly. MoE layers/router and finalize_model_grads read tp_dp_cp; cuda-graph capture reads tp_dp; the ProcessGroupCollection leaves them init=False by default. These edits extend the topology helper that landed with MM1; they are purely additive. The grid-args unit test travels with args.py. The existing topology unit test on main is unchanged. Validated in the 8-GPU 20L Nemotron VLM e2e (trains + checkpoint save/resume, lm loss 12.18->11.54 across resume). Signed-off-by: ykarnati --- examples/mimo/training/args.py | 253 ++++++++++++++++++ examples/mimo/training/topology.py | 11 +- .../models/mimo/test_mimo_hetero_grid_args.py | 152 +++++++++++ 3 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 examples/mimo/training/args.py create mode 100644 tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py diff --git a/examples/mimo/training/args.py b/examples/mimo/training/args.py new file mode 100644 index 00000000000..5ded106505b --- /dev/null +++ b/examples/mimo/training/args.py @@ -0,0 +1,253 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Hetero grid/topology CLI args + validation for the stock-args MIMO examples. + +This is PR-E2 of the NMFW-516 hetero-MIMO upstreaming effort. It contributes the +heterogeneous parallelism/topology argument group and its validation, designed +to compose with Megatron's *stock* argument system as an +``extra_args_provider`` and a post-``validate_args`` validation hook. + +Responsibilities (this PR only): + * :func:`add_hetero_grid_args` registers the per-module parallelism knobs + (``--encoder-*`` and ``--llm-*``) plus ``--llm-offset`` / ``--llm-only``. + These are namespaced so they never collide with stock parallelism flags + (``--tensor-model-parallel-size`` and friends are owned by the LLM grid but + expressed via the namespaced ``--llm-tp`` so the encoder grid can differ). + * :func:`validate_hetero_grid_args` ports the prototype's hetero invariants: + disjoint + covering rank spans, fan-out divisibility, MoE EP divisibility, + and the sample-vs-iter scheduler resolution. Call it AFTER stock + ``validate_args`` so tokenizer/preset-derived sizes are populated. + * :func:`build_module_grid_specs` maps parsed grid args onto the two + :class:`~examples.mimo.training.topology.ModuleGridSpec` s that + :func:`~examples.mimo.training.topology.create_topology` consumes. + +Arg-ownership note: ``--llm-ep`` and ``--llm-expt-tp`` are *parallelism* knobs +and live here, not in the model provider's ``add_model_provider_args``. The +provider still reads them via ``getattr(args, "llm_ep", ...)`` / +``getattr(args, "llm_expt_tp", ...)`` fallbacks, so the two PRs compose cleanly +once stacked. +""" + +from __future__ import annotations + +import argparse +import math +from typing import List + +from examples.mimo.training.topology import ModuleGridSpec +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY + +# Default name for the (single) vision encoder module grid. The E1 provider sets +# ``args.vision_encoder_key = "radio_encoder"``; we mirror that default here so +# the encoder ModuleGridSpec carries the same module name the provider/runtime +# look up. Resolved at spec-build time via ``getattr(args, "vision_encoder_key")``. +DEFAULT_ENCODER_MODULE_NAME = "radio_encoder" + + +def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """Register the hetero parallelism/topology arg group. + + Stock-hook compatible: returns the parser so it can be passed straight to + ``pretrain(extra_args_provider=...)`` or composed with the model-provider + args provider. Declares only namespaced ``--encoder-*`` / ``--llm-*`` knobs; + never re-declares a stock flag (``--tensor-model-parallel-size``, + ``--micro-batch-size``, ``--global-batch-size``, ``--num-experts``, ...). + """ + grid = parser.add_argument_group("hetero module grids") + + # Encoder grid placement + factorization. + grid.add_argument("--encoder-offset", type=int, default=0, + help="First global rank of the vision-encoder grid span.") + grid.add_argument("--encoder-tp", type=int, default=2, + help="Encoder tensor-model-parallel size.") + grid.add_argument("--encoder-cp", type=int, default=1, + help="Encoder context-parallel size (CP=1 only for now).") + grid.add_argument("--encoder-pp", type=int, default=1, + help="Encoder pipeline-model-parallel size.") + grid.add_argument("--encoder-dp", type=int, default=2, + help="Encoder data-parallel size.") + + # Language grid placement + factorization. + grid.add_argument("--llm-offset", type=int, default=4, + help="First global rank of the language grid span.") + grid.add_argument("--llm-tp", type=int, default=2, + help="Language tensor-model-parallel size.") + grid.add_argument("--llm-cp", type=int, default=1, + help="Language context-parallel size (CP=1 only for now).") + grid.add_argument("--llm-pp", type=int, default=1, + help="Language pipeline-model-parallel size.") + grid.add_argument("--llm-dp", type=int, default=2, + help="Language data-parallel size. Global batch is keyed on this.") + # MoE expert parallelism for the language grid. Relocated here from the E1 + # model provider's add_model_provider_args (these are topology knobs); the + # provider keeps a getattr(args, "llm_ep"/"llm_expt_tp", ...) fallback read. + grid.add_argument("--llm-ep", type=int, default=1, + help="Language expert-model-parallel size (MoE).") + grid.add_argument("--llm-expt-tp", type=int, default=None, + help="Language expert tensor-parallel size; defaults to 1 when unset " + "(experts default to TP=1; the 20L MoE recipe passes --llm-expt-tp 1).") + grid.add_argument("--llm-expt-dp", type=int, default=None, + help="Language expert data-parallel size; derived from the grid when unset.") + + grid.add_argument( + "--llm-only", + action="store_true", + help=( + "Run only the MIMO language module on the LLM grid. Keeps the MIMO " + "training/data path but creates no encoder ranks or bridge communicators; " + "requires --llm-offset 0 so the language grid covers WORLD_SIZE." + ), + ) + return parser + + +def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: + """Validate the disjoint-grid hetero layout. Returns ``(encoder_size, llm_size)``. + + Call AFTER stock ``validate_args`` (so ``micro_batch_size`` / ``num_experts`` + are populated) and after the model-provider preset (so ``num_experts`` reflects + the active provider). Ports the prototype invariants: + + * CP must be 1 on both grids (Phase-2 limitation). + * MoE: ``num_experts % llm_ep == 0`` (resolves PR-E1 open-Q3 -- the + EP-divisibility check belongs here, not in the provider). + * Global batch is keyed on ``llm_dp``: ``mbs * num_microbatches * llm_dp``. + * Sample-vs-iter scheduler resolution when ``--train-samples`` is present. + * ``--llm-only``: language grid must cover ``[0, world_size)`` exactly. + * Fan-out divisibility: ``mbs * llm_dp % encoder_dp == 0``. + * Encoder + LLM rank spans are DISJOINT and COVERING over ``world_size``. + """ + if args.encoder_cp != 1 or args.llm_cp != 1: + raise ValueError("hetero MIMO training currently supports CP=1 only") + + # MoE EP-divisibility (PR-E1 open-Q3 resolution). Stock arg is --num-experts, + # surfaced on args as num_experts; the prototype also used num_moe_experts. + num_experts = _num_experts(args) + if num_experts and num_experts % args.llm_ep != 0: + raise ValueError( + f"--num-experts ({num_experts}) must be divisible by --llm-ep ({args.llm_ep})" + ) + + # Sample-based scheduler resolution: derive --train-iters from --train-samples + # using the llm_dp-keyed global batch size. + if getattr(args, "train_samples", None) is not None: + derived_gbs = args.micro_batch_size * _num_microbatches(args) * args.llm_dp + gbs = args.global_batch_size if getattr(args, "global_batch_size", None) else derived_gbs + if gbs <= 0: + raise ValueError( + "--train-samples requires a positive derived/explicit --global-batch-size" + ) + args.train_iters = math.ceil(args.train_samples / gbs) + + llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp + + if args.llm_only: + if args.llm_offset != 0: + raise ValueError( + "--llm-only requires --llm-offset 0 so language ranks cover WORLD_SIZE" + ) + llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) + all_ranks = set(range(world_size)) + if llm_ranks != all_ranks: + raise ValueError( + "--llm-only requires the language grid to cover every torchrun rank exactly " + f"once; covered={sorted(llm_ranks)}, world={sorted(all_ranks)}" + ) + return 0, llm_size + + # Fan-out divisibility: the bridge splits (mbs * llm_dp) LLM lanes across + # encoder_dp encoder lanes; the split must be exact. + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError( + "--micro-batch-size * --llm-dp must be divisible by --encoder-dp " + f"(got {args.micro_batch_size} * {args.llm_dp} % {args.encoder_dp} != 0)" + ) + + encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp + encoder_ranks = set(range(args.encoder_offset, args.encoder_offset + encoder_size)) + llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) + all_ranks = set(range(world_size)) + + if not encoder_ranks.isdisjoint(llm_ranks): + raise ValueError( + "hetero MIMO expects disjoint module rank spans; " + f"spans overlap at {sorted(encoder_ranks & llm_ranks)}" + ) + if encoder_ranks | llm_ranks != all_ranks: + raise ValueError( + "The non-colocated module grids must cover every torchrun rank exactly once; " + f"covered={sorted(encoder_ranks | llm_ranks)}, world={sorted(all_ranks)}" + ) + + return encoder_size, llm_size + + +def build_module_grid_specs( + args: argparse.Namespace, world_size: int +) -> List[ModuleGridSpec]: + """Map parsed grid args onto the ModuleGridSpec list create_topology consumes. + + Returns ``[encoder_spec, language_spec]`` (or just ``[language_spec]`` when + ``--llm-only``). ``num_ranks`` is the ground truth ModuleGridSpec field; + ``dp`` / ``expt_dp`` are derived in ``ModuleGridSpec.__post_init__`` from the + tp/cp/pp/ep/expt_tp factorization, so we pass ``num_ranks = tp*cp*pp*dp`` and + let the dataclass re-derive dp (matching the supplied value) and expt_dp. + + Must be called after :func:`validate_hetero_grid_args` so the spans are known + to tile ``[0, world_size)``. + """ + encoder_size, llm_size = validate_hetero_grid_args(args, world_size) + + language_spec = ModuleGridSpec( + name=MIMO_LANGUAGE_MODULE_KEY, + num_ranks=llm_size, + tp=args.llm_tp, + cp=args.llm_cp, + pp=args.llm_pp, + ep=args.llm_ep, + rank_offset=args.llm_offset, + expt_tp=_resolve_expt_tp(args.llm_expt_tp, args.llm_tp), + ) + + if args.llm_only: + return [language_spec] + + encoder_name = getattr(args, "vision_encoder_key", None) or DEFAULT_ENCODER_MODULE_NAME + encoder_spec = ModuleGridSpec( + name=encoder_name, + num_ranks=encoder_size, + tp=args.encoder_tp, + cp=args.encoder_cp, + pp=args.encoder_pp, + ep=1, + rank_offset=args.encoder_offset, + expt_tp=1, + ) + return [encoder_spec, language_spec] + + +def _resolve_expt_tp(expt_tp, tp: int) -> int: + """Expert TP defaults to 1 when unset. + + Matches ModuleGridSpec's convention (experts default to TP=1, set explicitly + for MoE -- intentionally not Megatron's etp=tp). The 20L MoE layout (ep=4 over + 4 ranks) requires expt_tp=1: expt_tp*ep*pp must divide num_ranks. + """ + return 1 if expt_tp is None else expt_tp + + +def _num_experts(args: argparse.Namespace) -> int: + """Resolve MoE expert count from stock (--num-experts) or prototype args.""" + for attr in ("num_experts", "num_moe_experts"): + value = getattr(args, attr, None) + if value: + return int(value) + return 0 + + +def _num_microbatches(args: argparse.Namespace) -> int: + """Resolve the per-step microbatch count from whichever arg the loop exposes.""" + value = getattr(args, "num_microbatches", None) + if value: + return int(value) + return 1 diff --git a/examples/mimo/training/topology.py b/examples/mimo/training/topology.py index cf22de86627..03c38a41be2 100644 --- a/examples/mimo/training/topology.py +++ b/examples/mimo/training/topology.py @@ -132,7 +132,11 @@ def _build_grid(spec: ModuleGridSpec) -> HyperCommGrid: ) try: - for dims in (["tp"], ["cp"], ["pp"], ["dp"], ["dp", "cp"], ["tp", "cp"], ["tp", "pp"]): + for dims in ( + ["tp"], ["cp"], ["pp"], ["dp"], + ["dp", "cp"], ["tp", "cp"], ["tp", "pp"], + ["tp", "dp"], ["tp", "dp", "cp"], ["tp", "cp", "dp", "pp"], + ): grid.create_pg(dims) for dims in (["ep"], ["expt_tp"], ["expt_dp"], ["expt_tp", "ep"], ["expt_tp", "ep", "pp"]): grid.create_pg(dims, view=_EXPERT_VIEW) @@ -191,6 +195,11 @@ def pg_collection_from_grid( pgc.dp_cp = grid.get_pg(["dp", "cp"]) pgc.intra_dp_cp = pgc.dp_cp pgc.tp_cp = grid.get_pg(["tp", "cp"]) + # MoE layers/router and finalize_model_grads read tp_dp_cp (tensor+data+context + # parallel group); cuda-graph capture reads tp_dp. Set them explicitly since the + # ProcessGroupCollection leaves them init=False. + pgc.tp_dp = grid.get_pg(["tp", "dp"]) + pgc.tp_dp_cp = grid.get_pg(["tp", "dp", "cp"]) pgc.mp = grid.get_pg(["tp", "pp"]) pgc.ep = grid.get_pg("ep", view=_EXPERT_VIEW) pgc.expt_tp = grid.get_pg("expt_tp", view=_EXPERT_VIEW) diff --git a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py new file mode 100644 index 00000000000..840e93e14a6 --- /dev/null +++ b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py @@ -0,0 +1,152 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Pure-args (no-GPU) tests for the hetero grid arg group + validation. + +Covers: + * the canonical 8-GPU 20L layout parses and validates OK + (encoder tp2/pp1/dp2 = ranks 0-3, llm offset4 tp2/pp1/dp2/ep4 = ranks 4-7); + * disjoint / covering violations raise; + * fan-out divisibility + EP-divisibility checks raise on bad inputs; + * build_module_grid_specs yields the correct num_ranks / rank_offset. + +These run without torch.distributed -- they only exercise argparse + the +validation/spec-mapping helpers, so they are safe in the CPU CI lane. +""" + +from __future__ import annotations + +import argparse + +import pytest + +from examples.mimo.training.args import ( + add_hetero_grid_args, + build_module_grid_specs, + validate_hetero_grid_args, +) +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY + +WORLD_SIZE_8 = 8 + + +def _parse(argv): + """Parse only the hetero grid args from a token list.""" + parser = argparse.ArgumentParser() + add_hetero_grid_args(parser) + return parser.parse_args(argv) + + +def _layout_8gpu_20l(**overrides): + """Canonical 8-GPU layout: encoder 0-3 (tp2/pp1/dp2), llm 4-7 (tp2/pp1/dp2/ep4).""" + argv = [ + "--encoder-offset", "0", + "--encoder-tp", "2", + "--encoder-pp", "1", + "--encoder-dp", "2", + "--llm-offset", "4", + "--llm-tp", "2", + "--llm-pp", "1", + "--llm-dp", "2", + "--llm-ep", "4", + ] + args = _parse(argv) + # Stock args the validator reads but the grid parser does not own. + args.micro_batch_size = 1 + args.num_microbatches = 2 + args.global_batch_size = None + args.train_samples = None + args.num_experts = 128 + args.vision_encoder_key = "radio_encoder" + for key, value in overrides.items(): + setattr(args, key, value) + return args + + +def test_canonical_8gpu_20l_layout_validates(): + args = _layout_8gpu_20l() + encoder_size, llm_size = validate_hetero_grid_args(args, WORLD_SIZE_8) + assert encoder_size == 4 # tp2 * cp1 * pp1 * dp2 + assert llm_size == 4 # tp2 * cp1 * pp1 * dp2 + + +def test_module_grid_specs_mapping(): + args = _layout_8gpu_20l() + specs = build_module_grid_specs(args, WORLD_SIZE_8) + assert len(specs) == 2 + encoder_spec, language_spec = specs + + assert encoder_spec.name == "radio_encoder" + assert encoder_spec.num_ranks == 4 + assert encoder_spec.rank_offset == 0 + assert encoder_spec.tp == 2 + assert encoder_spec.dp == 2 # derived in __post_init__: 4 // (2*1*1) + + assert language_spec.name == MIMO_LANGUAGE_MODULE_KEY + assert language_spec.num_ranks == 4 + assert language_spec.rank_offset == 4 + assert language_spec.tp == 2 + assert language_spec.ep == 4 + assert language_spec.dp == 2 # 4 // (2*1*1) + # expt_tp defaults to 1 when --llm-expt-tp unset (the 20L script passes --llm-expt-tp 1); + # ep=4 over 4 ranks requires expt_tp=1 so expt_tp*ep*pp divides num_ranks. + assert language_spec.expt_tp == 1 + + +def test_overlapping_spans_raise(): + # llm-offset 2 makes llm ranks {2,3,4,5} overlap encoder ranks {0,1,2,3}. + args = _layout_8gpu_20l(llm_offset=2) + with pytest.raises(ValueError, match="disjoint"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_non_covering_spans_raise(): + # encoder 0-3 + llm 4-7 cover only 8 ranks; declare world_size 10 -> gap. + args = _layout_8gpu_20l() + with pytest.raises(ValueError, match="cover every torchrun rank"): + validate_hetero_grid_args(args, 10) + + +def test_fanout_divisibility_raises(): + # mbs(1) * llm_dp(2) = 2 not divisible by encoder_dp(3); also keep spans valid + # by widening encoder to tp1/dp3 isn't tiling-clean, so just trigger the + # fan-out check before the span check by an indivisible encoder_dp. + args = _layout_8gpu_20l(encoder_dp=3, micro_batch_size=1, llm_dp=2) + with pytest.raises(ValueError, match="divisible by --encoder-dp"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_ep_divisibility_raises(): + # num_experts 128 not divisible by llm_ep 3. + args = _layout_8gpu_20l(llm_ep=3, num_experts=128) + with pytest.raises(ValueError, match="divisible by --llm-ep"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_cp_must_be_one(): + args = _layout_8gpu_20l(llm_cp=2) + with pytest.raises(ValueError, match="CP=1 only"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_llm_only_requires_offset_zero(): + args = _layout_8gpu_20l(llm_only=True, llm_offset=4) + with pytest.raises(ValueError, match="--llm-only requires --llm-offset 0"): + validate_hetero_grid_args(args, WORLD_SIZE_8) + + +def test_llm_only_covers_world(): + # llm tp2/pp1/dp2 = 4 ranks at offset 0; world_size 4 -> covers exactly. + args = _layout_8gpu_20l(llm_only=True, llm_offset=0, llm_ep=2, num_experts=128) + encoder_size, llm_size = validate_hetero_grid_args(args, 4) + assert encoder_size == 0 + assert llm_size == 4 + specs = build_module_grid_specs(args, 4) + assert len(specs) == 1 + assert specs[0].name == MIMO_LANGUAGE_MODULE_KEY + + +def test_train_samples_resolves_iters(): + # gbs = mbs(1) * num_microbatches(2) * llm_dp(2) = 4; 17 samples -> ceil(17/4)=5. + args = _layout_8gpu_20l(train_samples=17, train_iters=999) + validate_hetero_grid_args(args, WORLD_SIZE_8) + assert args.train_iters == 5 From c3f466d97bfb799515a6abe15bad406a9a84d8e4 Mon Sep 17 00:00:00 2001 From: ykarnati Date: Thu, 18 Jun 2026 17:30:42 -0700 Subject: [PATCH 2/6] mimo example: address review comments on hetero grid args Trim verbose module/function docstrings and comments, remove the unused --encoder-offset arg (the encoder span always starts at rank 0), inline the trivial expert-TP default, read MoE expert count from stock --num-experts only, and rename the local grid specs to language_grid_spec / encoder_grid_spec. Slim the pure-args tests to the value-asserting cases. Co-Authored-By: Claude Opus 4.8 Signed-off-by: ykarnati --- examples/mimo/training/args.py | 106 ++++-------------- examples/mimo/training/topology.py | 4 +- .../models/mimo/test_mimo_hetero_grid_args.py | 75 ++++--------- 3 files changed, 41 insertions(+), 144 deletions(-) diff --git a/examples/mimo/training/args.py b/examples/mimo/training/args.py index 5ded106505b..e106a53041f 100644 --- a/examples/mimo/training/args.py +++ b/examples/mimo/training/args.py @@ -1,32 +1,6 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -"""Hetero grid/topology CLI args + validation for the stock-args MIMO examples. - -This is PR-E2 of the NMFW-516 hetero-MIMO upstreaming effort. It contributes the -heterogeneous parallelism/topology argument group and its validation, designed -to compose with Megatron's *stock* argument system as an -``extra_args_provider`` and a post-``validate_args`` validation hook. - -Responsibilities (this PR only): - * :func:`add_hetero_grid_args` registers the per-module parallelism knobs - (``--encoder-*`` and ``--llm-*``) plus ``--llm-offset`` / ``--llm-only``. - These are namespaced so they never collide with stock parallelism flags - (``--tensor-model-parallel-size`` and friends are owned by the LLM grid but - expressed via the namespaced ``--llm-tp`` so the encoder grid can differ). - * :func:`validate_hetero_grid_args` ports the prototype's hetero invariants: - disjoint + covering rank spans, fan-out divisibility, MoE EP divisibility, - and the sample-vs-iter scheduler resolution. Call it AFTER stock - ``validate_args`` so tokenizer/preset-derived sizes are populated. - * :func:`build_module_grid_specs` maps parsed grid args onto the two - :class:`~examples.mimo.training.topology.ModuleGridSpec` s that - :func:`~examples.mimo.training.topology.create_topology` consumes. - -Arg-ownership note: ``--llm-ep`` and ``--llm-expt-tp`` are *parallelism* knobs -and live here, not in the model provider's ``add_model_provider_args``. The -provider still reads them via ``getattr(args, "llm_ep", ...)`` / -``getattr(args, "llm_expt_tp", ...)`` fallbacks, so the two PRs compose cleanly -once stacked. -""" +"""Hetero grid/topology CLI args + validation for the MIMO example.""" from __future__ import annotations @@ -37,27 +11,15 @@ from examples.mimo.training.topology import ModuleGridSpec from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY -# Default name for the (single) vision encoder module grid. The E1 provider sets -# ``args.vision_encoder_key = "radio_encoder"``; we mirror that default here so -# the encoder ModuleGridSpec carries the same module name the provider/runtime -# look up. Resolved at spec-build time via ``getattr(args, "vision_encoder_key")``. +# Default encoder grid module name (single-sourced with radio_encoder.py in #5397). DEFAULT_ENCODER_MODULE_NAME = "radio_encoder" def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - """Register the hetero parallelism/topology arg group. - - Stock-hook compatible: returns the parser so it can be passed straight to - ``pretrain(extra_args_provider=...)`` or composed with the model-provider - args provider. Declares only namespaced ``--encoder-*`` / ``--llm-*`` knobs; - never re-declares a stock flag (``--tensor-model-parallel-size``, - ``--micro-batch-size``, ``--global-batch-size``, ``--num-experts``, ...). - """ + """Register the hetero per-module parallelism args (--encoder-*/--llm-*).""" grid = parser.add_argument_group("hetero module grids") - # Encoder grid placement + factorization. - grid.add_argument("--encoder-offset", type=int, default=0, - help="First global rank of the vision-encoder grid span.") + # Encoder grid factorization (the encoder span always starts at rank 0). grid.add_argument("--encoder-tp", type=int, default=2, help="Encoder tensor-model-parallel size.") grid.add_argument("--encoder-cp", type=int, default=1, @@ -78,9 +40,7 @@ def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentPa help="Language pipeline-model-parallel size.") grid.add_argument("--llm-dp", type=int, default=2, help="Language data-parallel size. Global batch is keyed on this.") - # MoE expert parallelism for the language grid. Relocated here from the E1 - # model provider's add_model_provider_args (these are topology knobs); the - # provider keeps a getattr(args, "llm_ep"/"llm_expt_tp", ...) fallback read. + # MoE expert parallelism for the language grid. grid.add_argument("--llm-ep", type=int, default=1, help="Language expert-model-parallel size (MoE).") grid.add_argument("--llm-expt-tp", type=int, default=None, @@ -102,26 +62,11 @@ def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentPa def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: - """Validate the disjoint-grid hetero layout. Returns ``(encoder_size, llm_size)``. - - Call AFTER stock ``validate_args`` (so ``micro_batch_size`` / ``num_experts`` - are populated) and after the model-provider preset (so ``num_experts`` reflects - the active provider). Ports the prototype invariants: - - * CP must be 1 on both grids (Phase-2 limitation). - * MoE: ``num_experts % llm_ep == 0`` (resolves PR-E1 open-Q3 -- the - EP-divisibility check belongs here, not in the provider). - * Global batch is keyed on ``llm_dp``: ``mbs * num_microbatches * llm_dp``. - * Sample-vs-iter scheduler resolution when ``--train-samples`` is present. - * ``--llm-only``: language grid must cover ``[0, world_size)`` exactly. - * Fan-out divisibility: ``mbs * llm_dp % encoder_dp == 0``. - * Encoder + LLM rank spans are DISJOINT and COVERING over ``world_size``. - """ + """Validate the disjoint hetero grid layout; returns ``(encoder_size, llm_size)``.""" if args.encoder_cp != 1 or args.llm_cp != 1: raise ValueError("hetero MIMO training currently supports CP=1 only") - # MoE EP-divisibility (PR-E1 open-Q3 resolution). Stock arg is --num-experts, - # surfaced on args as num_experts; the prototype also used num_moe_experts. + # MoE expert count must divide evenly across the language grid's expert parallelism. num_experts = _num_experts(args) if num_experts and num_experts % args.llm_ep != 0: raise ValueError( @@ -164,7 +109,7 @@ def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tupl ) encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp - encoder_ranks = set(range(args.encoder_offset, args.encoder_offset + encoder_size)) + encoder_ranks = set(range(encoder_size)) # encoder span always starts at rank 0 llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) all_ranks = set(range(world_size)) @@ -187,8 +132,8 @@ def build_module_grid_specs( ) -> List[ModuleGridSpec]: """Map parsed grid args onto the ModuleGridSpec list create_topology consumes. - Returns ``[encoder_spec, language_spec]`` (or just ``[language_spec]`` when - ``--llm-only``). ``num_ranks`` is the ground truth ModuleGridSpec field; + Returns ``[encoder_grid_spec, language_grid_spec]`` (or just the language spec + when ``--llm-only``). ``num_ranks`` is the ground truth ModuleGridSpec field; ``dp`` / ``expt_dp`` are derived in ``ModuleGridSpec.__post_init__`` from the tp/cp/pp/ep/expt_tp factorization, so we pass ``num_ranks = tp*cp*pp*dp`` and let the dataclass re-derive dp (matching the supplied value) and expt_dp. @@ -198,7 +143,7 @@ def build_module_grid_specs( """ encoder_size, llm_size = validate_hetero_grid_args(args, world_size) - language_spec = ModuleGridSpec( + language_grid_spec = ModuleGridSpec( name=MIMO_LANGUAGE_MODULE_KEY, num_ranks=llm_size, tp=args.llm_tp, @@ -206,43 +151,30 @@ def build_module_grid_specs( pp=args.llm_pp, ep=args.llm_ep, rank_offset=args.llm_offset, - expt_tp=_resolve_expt_tp(args.llm_expt_tp, args.llm_tp), + expt_tp=args.llm_expt_tp or 1, ) if args.llm_only: - return [language_spec] + return [language_grid_spec] encoder_name = getattr(args, "vision_encoder_key", None) or DEFAULT_ENCODER_MODULE_NAME - encoder_spec = ModuleGridSpec( + encoder_grid_spec = ModuleGridSpec( name=encoder_name, num_ranks=encoder_size, tp=args.encoder_tp, cp=args.encoder_cp, pp=args.encoder_pp, ep=1, - rank_offset=args.encoder_offset, + rank_offset=0, expt_tp=1, ) - return [encoder_spec, language_spec] - - -def _resolve_expt_tp(expt_tp, tp: int) -> int: - """Expert TP defaults to 1 when unset. - - Matches ModuleGridSpec's convention (experts default to TP=1, set explicitly - for MoE -- intentionally not Megatron's etp=tp). The 20L MoE layout (ep=4 over - 4 ranks) requires expt_tp=1: expt_tp*ep*pp must divide num_ranks. - """ - return 1 if expt_tp is None else expt_tp + return [encoder_grid_spec, language_grid_spec] def _num_experts(args: argparse.Namespace) -> int: - """Resolve MoE expert count from stock (--num-experts) or prototype args.""" - for attr in ("num_experts", "num_moe_experts"): - value = getattr(args, attr, None) - if value: - return int(value) - return 0 + """Resolve MoE expert count from the stock --num-experts arg.""" + value = getattr(args, "num_experts", None) + return int(value) if value else 0 def _num_microbatches(args: argparse.Namespace) -> int: diff --git a/examples/mimo/training/topology.py b/examples/mimo/training/topology.py index 03c38a41be2..7a79f0dcad6 100644 --- a/examples/mimo/training/topology.py +++ b/examples/mimo/training/topology.py @@ -195,9 +195,7 @@ def pg_collection_from_grid( pgc.dp_cp = grid.get_pg(["dp", "cp"]) pgc.intra_dp_cp = pgc.dp_cp pgc.tp_cp = grid.get_pg(["tp", "cp"]) - # MoE layers/router and finalize_model_grads read tp_dp_cp (tensor+data+context - # parallel group); cuda-graph capture reads tp_dp. Set them explicitly since the - # ProcessGroupCollection leaves them init=False. + # MoE/finalize_model_grads read tp_dp_cp; cuda-graph capture reads tp_dp. pgc.tp_dp = grid.get_pg(["tp", "dp"]) pgc.tp_dp_cp = grid.get_pg(["tp", "dp", "cp"]) pgc.mp = grid.get_pg(["tp", "pp"]) diff --git a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py index 840e93e14a6..c87007ef76e 100644 --- a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py +++ b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py @@ -1,17 +1,6 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -"""Pure-args (no-GPU) tests for the hetero grid arg group + validation. - -Covers: - * the canonical 8-GPU 20L layout parses and validates OK - (encoder tp2/pp1/dp2 = ranks 0-3, llm offset4 tp2/pp1/dp2/ep4 = ranks 4-7); - * disjoint / covering violations raise; - * fan-out divisibility + EP-divisibility checks raise on bad inputs; - * build_module_grid_specs yields the correct num_ranks / rank_offset. - -These run without torch.distributed -- they only exercise argparse + the -validation/spec-mapping helpers, so they are safe in the CPU CI lane. -""" +"""Pure-args (no-GPU) tests for the hetero grid arg group + validation.""" from __future__ import annotations @@ -38,17 +27,10 @@ def _parse(argv): def _layout_8gpu_20l(**overrides): """Canonical 8-GPU layout: encoder 0-3 (tp2/pp1/dp2), llm 4-7 (tp2/pp1/dp2/ep4).""" - argv = [ - "--encoder-offset", "0", - "--encoder-tp", "2", - "--encoder-pp", "1", - "--encoder-dp", "2", - "--llm-offset", "4", - "--llm-tp", "2", - "--llm-pp", "1", - "--llm-dp", "2", - "--llm-ep", "4", - ] + argv = ( + "--encoder-tp 2 --encoder-pp 1 --encoder-dp 2 " + "--llm-offset 4 --llm-tp 2 --llm-pp 1 --llm-dp 2 --llm-ep 4" + ).split() args = _parse(argv) # Stock args the validator reads but the grid parser does not own. args.micro_batch_size = 1 @@ -62,34 +44,22 @@ def _layout_8gpu_20l(**overrides): return args -def test_canonical_8gpu_20l_layout_validates(): +def test_canonical_layout_validates_and_maps_specs(): args = _layout_8gpu_20l() encoder_size, llm_size = validate_hetero_grid_args(args, WORLD_SIZE_8) - assert encoder_size == 4 # tp2 * cp1 * pp1 * dp2 - assert llm_size == 4 # tp2 * cp1 * pp1 * dp2 + assert (encoder_size, llm_size) == (4, 4) - -def test_module_grid_specs_mapping(): - args = _layout_8gpu_20l() - specs = build_module_grid_specs(args, WORLD_SIZE_8) - assert len(specs) == 2 - encoder_spec, language_spec = specs - - assert encoder_spec.name == "radio_encoder" - assert encoder_spec.num_ranks == 4 - assert encoder_spec.rank_offset == 0 - assert encoder_spec.tp == 2 - assert encoder_spec.dp == 2 # derived in __post_init__: 4 // (2*1*1) - - assert language_spec.name == MIMO_LANGUAGE_MODULE_KEY - assert language_spec.num_ranks == 4 - assert language_spec.rank_offset == 4 - assert language_spec.tp == 2 - assert language_spec.ep == 4 - assert language_spec.dp == 2 # 4 // (2*1*1) - # expt_tp defaults to 1 when --llm-expt-tp unset (the 20L script passes --llm-expt-tp 1); - # ep=4 over 4 ranks requires expt_tp=1 so expt_tp*ep*pp divides num_ranks. - assert language_spec.expt_tp == 1 + encoder_grid_spec, language_grid_spec = build_module_grid_specs(args, WORLD_SIZE_8) + assert encoder_grid_spec.name == "radio_encoder" + assert encoder_grid_spec.num_ranks == 4 + assert encoder_grid_spec.rank_offset == 0 # encoder span always starts at rank 0 + assert encoder_grid_spec.dp == 2 # derived: 4 // (tp2 * cp1 * pp1) + assert language_grid_spec.name == MIMO_LANGUAGE_MODULE_KEY + assert language_grid_spec.num_ranks == 4 + assert language_grid_spec.rank_offset == 4 + assert language_grid_spec.dp == 2 + # expt_tp defaults to 1 when --llm-expt-tp unset (ep=4 over 4 ranks needs expt_tp=1). + assert language_grid_spec.expt_tp == 1 def test_overlapping_spans_raise(): @@ -107,9 +77,7 @@ def test_non_covering_spans_raise(): def test_fanout_divisibility_raises(): - # mbs(1) * llm_dp(2) = 2 not divisible by encoder_dp(3); also keep spans valid - # by widening encoder to tp1/dp3 isn't tiling-clean, so just trigger the - # fan-out check before the span check by an indivisible encoder_dp. + # mbs(1) * llm_dp(2) = 2 not divisible by encoder_dp(3). args = _layout_8gpu_20l(encoder_dp=3, micro_batch_size=1, llm_dp=2) with pytest.raises(ValueError, match="divisible by --encoder-dp"): validate_hetero_grid_args(args, WORLD_SIZE_8) @@ -135,11 +103,10 @@ def test_llm_only_requires_offset_zero(): def test_llm_only_covers_world(): - # llm tp2/pp1/dp2 = 4 ranks at offset 0; world_size 4 -> covers exactly. + # llm tp2/pp1/dp2 = 4 ranks at offset 0; world_size 4 -> covers exactly, no encoder spec. args = _layout_8gpu_20l(llm_only=True, llm_offset=0, llm_ep=2, num_experts=128) encoder_size, llm_size = validate_hetero_grid_args(args, 4) - assert encoder_size == 0 - assert llm_size == 4 + assert (encoder_size, llm_size) == (0, 4) specs = build_module_grid_specs(args, 4) assert len(specs) == 1 assert specs[0].name == MIMO_LANGUAGE_MODULE_KEY From 82e07c72b6df7ababc01bc1c4cb1881613ad2f08 Mon Sep 17 00:00:00 2001 From: ykarnati Date: Thu, 18 Jun 2026 17:32:52 -0700 Subject: [PATCH 3/6] mimo example: derive train_iters from --global-batch-size directly Drop the _num_microbatches helper and key the --train-samples to --train-iters conversion off the explicit --global-batch-size. num_microbatches is derived by the stock calculator (gbs / (mbs * llm_dp)), so the redundant --num-microbatches read is removed here. The conversion stays: stock validate_args does not derive train_iters from train_samples and the MIMO loop reads args.train_iters. Co-Authored-By: Claude Opus 4.8 Signed-off-by: ykarnati --- examples/mimo/training/args.py | 20 ++++--------------- .../models/mimo/test_mimo_hetero_grid_args.py | 5 ++--- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/examples/mimo/training/args.py b/examples/mimo/training/args.py index e106a53041f..9b68a3a5395 100644 --- a/examples/mimo/training/args.py +++ b/examples/mimo/training/args.py @@ -73,15 +73,11 @@ def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tupl f"--num-experts ({num_experts}) must be divisible by --llm-ep ({args.llm_ep})" ) - # Sample-based scheduler resolution: derive --train-iters from --train-samples - # using the llm_dp-keyed global batch size. + # Sample-based scheduler resolution: derive --train-iters from --train-samples. if getattr(args, "train_samples", None) is not None: - derived_gbs = args.micro_batch_size * _num_microbatches(args) * args.llm_dp - gbs = args.global_batch_size if getattr(args, "global_batch_size", None) else derived_gbs - if gbs <= 0: - raise ValueError( - "--train-samples requires a positive derived/explicit --global-batch-size" - ) + gbs = getattr(args, "global_batch_size", None) + if not gbs or gbs <= 0: + raise ValueError("--train-samples requires a positive --global-batch-size") args.train_iters = math.ceil(args.train_samples / gbs) llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp @@ -175,11 +171,3 @@ def _num_experts(args: argparse.Namespace) -> int: """Resolve MoE expert count from the stock --num-experts arg.""" value = getattr(args, "num_experts", None) return int(value) if value else 0 - - -def _num_microbatches(args: argparse.Namespace) -> int: - """Resolve the per-step microbatch count from whichever arg the loop exposes.""" - value = getattr(args, "num_microbatches", None) - if value: - return int(value) - return 1 diff --git a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py index c87007ef76e..22b0ea9db66 100644 --- a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py +++ b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py @@ -34,7 +34,6 @@ def _layout_8gpu_20l(**overrides): args = _parse(argv) # Stock args the validator reads but the grid parser does not own. args.micro_batch_size = 1 - args.num_microbatches = 2 args.global_batch_size = None args.train_samples = None args.num_experts = 128 @@ -113,7 +112,7 @@ def test_llm_only_covers_world(): def test_train_samples_resolves_iters(): - # gbs = mbs(1) * num_microbatches(2) * llm_dp(2) = 4; 17 samples -> ceil(17/4)=5. - args = _layout_8gpu_20l(train_samples=17, train_iters=999) + # gbs 4, 17 samples -> ceil(17/4) = 5. + args = _layout_8gpu_20l(train_samples=17, train_iters=999, global_batch_size=4) validate_hetero_grid_args(args, WORLD_SIZE_8) assert args.train_iters == 5 From e1af6e3e8349f2b274d101c124904bed954c6a5d Mon Sep 17 00:00:00 2001 From: ykarnati Date: Thu, 18 Jun 2026 17:56:03 -0700 Subject: [PATCH 4/6] mimo example: take encoder module name as a build_module_grid_specs param The caller (model provider) owns the encoder module name. Drop the duplicate DEFAULT_ENCODER_MODULE_NAME constant and the dead vision_encoder_key getattr; RADIO_ENCODER_MODULE_NAME (radio_encoder.py) is now the single source. Co-Authored-By: Claude Opus 4.8 Signed-off-by: ykarnati --- examples/mimo/training/args.py | 11 ++++------- .../models/mimo/test_mimo_hetero_grid_args.py | 7 ++++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/examples/mimo/training/args.py b/examples/mimo/training/args.py index 9b68a3a5395..5af6b74fcfa 100644 --- a/examples/mimo/training/args.py +++ b/examples/mimo/training/args.py @@ -11,9 +11,6 @@ from examples.mimo.training.topology import ModuleGridSpec from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY -# Default encoder grid module name (single-sourced with radio_encoder.py in #5397). -DEFAULT_ENCODER_MODULE_NAME = "radio_encoder" - def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: """Register the hetero per-module parallelism args (--encoder-*/--llm-*).""" @@ -124,12 +121,13 @@ def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tupl def build_module_grid_specs( - args: argparse.Namespace, world_size: int + args: argparse.Namespace, world_size: int, encoder_module_name: str ) -> List[ModuleGridSpec]: """Map parsed grid args onto the ModuleGridSpec list create_topology consumes. Returns ``[encoder_grid_spec, language_grid_spec]`` (or just the language spec - when ``--llm-only``). ``num_ranks`` is the ground truth ModuleGridSpec field; + when ``--llm-only``). The caller supplies ``encoder_module_name`` (the model + provider owns it). ``num_ranks`` is the ground truth ModuleGridSpec field; ``dp`` / ``expt_dp`` are derived in ``ModuleGridSpec.__post_init__`` from the tp/cp/pp/ep/expt_tp factorization, so we pass ``num_ranks = tp*cp*pp*dp`` and let the dataclass re-derive dp (matching the supplied value) and expt_dp. @@ -153,9 +151,8 @@ def build_module_grid_specs( if args.llm_only: return [language_grid_spec] - encoder_name = getattr(args, "vision_encoder_key", None) or DEFAULT_ENCODER_MODULE_NAME encoder_grid_spec = ModuleGridSpec( - name=encoder_name, + name=encoder_module_name, num_ranks=encoder_size, tp=args.encoder_tp, cp=args.encoder_cp, diff --git a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py index 22b0ea9db66..f3c8f89e085 100644 --- a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py +++ b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py @@ -37,7 +37,6 @@ def _layout_8gpu_20l(**overrides): args.global_batch_size = None args.train_samples = None args.num_experts = 128 - args.vision_encoder_key = "radio_encoder" for key, value in overrides.items(): setattr(args, key, value) return args @@ -48,7 +47,9 @@ def test_canonical_layout_validates_and_maps_specs(): encoder_size, llm_size = validate_hetero_grid_args(args, WORLD_SIZE_8) assert (encoder_size, llm_size) == (4, 4) - encoder_grid_spec, language_grid_spec = build_module_grid_specs(args, WORLD_SIZE_8) + encoder_grid_spec, language_grid_spec = build_module_grid_specs( + args, WORLD_SIZE_8, encoder_module_name="radio_encoder" + ) assert encoder_grid_spec.name == "radio_encoder" assert encoder_grid_spec.num_ranks == 4 assert encoder_grid_spec.rank_offset == 0 # encoder span always starts at rank 0 @@ -106,7 +107,7 @@ def test_llm_only_covers_world(): args = _layout_8gpu_20l(llm_only=True, llm_offset=0, llm_ep=2, num_experts=128) encoder_size, llm_size = validate_hetero_grid_args(args, 4) assert (encoder_size, llm_size) == (0, 4) - specs = build_module_grid_specs(args, 4) + specs = build_module_grid_specs(args, 4, encoder_module_name="radio_encoder") assert len(specs) == 1 assert specs[0].name == MIMO_LANGUAGE_MODULE_KEY From e3a0b7c5a355a5488a6fc8ae43e7aa56e09c348a Mon Sep 17 00:00:00 2001 From: ykarnati Date: Thu, 18 Jun 2026 18:24:15 -0700 Subject: [PATCH 5/6] mimo example: drop train_samples->train_iters from grid validation The conversion belongs to the training loop, not grid-layout validation. Stock update_train_iters owns it; the MIMO entry invokes it. validate_hetero_grid_args is now purely about grid layout. Also trim the build_module_grid_specs docstring. Co-Authored-By: Claude Opus 4.8 Signed-off-by: ykarnati --- examples/mimo/training/args.py | 21 +------------------ .../models/mimo/test_mimo_hetero_grid_args.py | 9 -------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/examples/mimo/training/args.py b/examples/mimo/training/args.py index 5af6b74fcfa..03bc088b94f 100644 --- a/examples/mimo/training/args.py +++ b/examples/mimo/training/args.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse -import math from typing import List from examples.mimo.training.topology import ModuleGridSpec @@ -70,13 +69,6 @@ def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tupl f"--num-experts ({num_experts}) must be divisible by --llm-ep ({args.llm_ep})" ) - # Sample-based scheduler resolution: derive --train-iters from --train-samples. - if getattr(args, "train_samples", None) is not None: - gbs = getattr(args, "global_batch_size", None) - if not gbs or gbs <= 0: - raise ValueError("--train-samples requires a positive --global-batch-size") - args.train_iters = math.ceil(args.train_samples / gbs) - llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp if args.llm_only: @@ -123,18 +115,7 @@ def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tupl def build_module_grid_specs( args: argparse.Namespace, world_size: int, encoder_module_name: str ) -> List[ModuleGridSpec]: - """Map parsed grid args onto the ModuleGridSpec list create_topology consumes. - - Returns ``[encoder_grid_spec, language_grid_spec]`` (or just the language spec - when ``--llm-only``). The caller supplies ``encoder_module_name`` (the model - provider owns it). ``num_ranks`` is the ground truth ModuleGridSpec field; - ``dp`` / ``expt_dp`` are derived in ``ModuleGridSpec.__post_init__`` from the - tp/cp/pp/ep/expt_tp factorization, so we pass ``num_ranks = tp*cp*pp*dp`` and - let the dataclass re-derive dp (matching the supplied value) and expt_dp. - - Must be called after :func:`validate_hetero_grid_args` so the spans are known - to tile ``[0, world_size)``. - """ + """Map grid args to the ModuleGridSpec list create_topology consumes; caller supplies encoder_module_name.""" encoder_size, llm_size = validate_hetero_grid_args(args, world_size) language_grid_spec = ModuleGridSpec( diff --git a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py index f3c8f89e085..fa6b6589e1f 100644 --- a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py +++ b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py @@ -34,8 +34,6 @@ def _layout_8gpu_20l(**overrides): args = _parse(argv) # Stock args the validator reads but the grid parser does not own. args.micro_batch_size = 1 - args.global_batch_size = None - args.train_samples = None args.num_experts = 128 for key, value in overrides.items(): setattr(args, key, value) @@ -110,10 +108,3 @@ def test_llm_only_covers_world(): specs = build_module_grid_specs(args, 4, encoder_module_name="radio_encoder") assert len(specs) == 1 assert specs[0].name == MIMO_LANGUAGE_MODULE_KEY - - -def test_train_samples_resolves_iters(): - # gbs 4, 17 samples -> ceil(17/4) = 5. - args = _layout_8gpu_20l(train_samples=17, train_iters=999, global_batch_size=4) - validate_hetero_grid_args(args, WORLD_SIZE_8) - assert args.train_iters == 5 From 4f0a1f9adc0796e7728e3ad1bd3e24a705fb03c6 Mon Sep 17 00:00:00 2001 From: ykarnati Date: Mon, 22 Jun 2026 23:36:50 -0700 Subject: [PATCH 6/6] mimo example: narrow hetero grid args to supported surface Signed-off-by: ykarnati --- examples/mimo/training/args.py | 20 +++++++------------ examples/mimo/training/topology.py | 1 - .../models/mimo/test_mimo_hetero_grid_args.py | 17 ++++++++++++---- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/examples/mimo/training/args.py b/examples/mimo/training/args.py index 03bc088b94f..e1d9e2116f7 100644 --- a/examples/mimo/training/args.py +++ b/examples/mimo/training/args.py @@ -12,16 +12,12 @@ def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - """Register the hetero per-module parallelism args (--encoder-*/--llm-*).""" + """Register hetero parallelism args for the single-encoder MIMO example.""" grid = parser.add_argument_group("hetero module grids") - # Encoder grid factorization (the encoder span always starts at rank 0). + # Single encoder grid; CP/PP stay fixed at 1. grid.add_argument("--encoder-tp", type=int, default=2, help="Encoder tensor-model-parallel size.") - grid.add_argument("--encoder-cp", type=int, default=1, - help="Encoder context-parallel size (CP=1 only for now).") - grid.add_argument("--encoder-pp", type=int, default=1, - help="Encoder pipeline-model-parallel size.") grid.add_argument("--encoder-dp", type=int, default=2, help="Encoder data-parallel size.") @@ -42,8 +38,6 @@ def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentPa grid.add_argument("--llm-expt-tp", type=int, default=None, help="Language expert tensor-parallel size; defaults to 1 when unset " "(experts default to TP=1; the 20L MoE recipe passes --llm-expt-tp 1).") - grid.add_argument("--llm-expt-dp", type=int, default=None, - help="Language expert data-parallel size; derived from the grid when unset.") grid.add_argument( "--llm-only", @@ -59,7 +53,7 @@ def add_hetero_grid_args(parser: argparse.ArgumentParser) -> argparse.ArgumentPa def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: """Validate the disjoint hetero grid layout; returns ``(encoder_size, llm_size)``.""" - if args.encoder_cp != 1 or args.llm_cp != 1: + if args.llm_cp != 1: raise ValueError("hetero MIMO training currently supports CP=1 only") # MoE expert count must divide evenly across the language grid's expert parallelism. @@ -93,7 +87,7 @@ def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tupl f"(got {args.micro_batch_size} * {args.llm_dp} % {args.encoder_dp} != 0)" ) - encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp + encoder_size = args.encoder_tp * args.encoder_dp encoder_ranks = set(range(encoder_size)) # encoder span always starts at rank 0 llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) all_ranks = set(range(world_size)) @@ -115,7 +109,7 @@ def validate_hetero_grid_args(args: argparse.Namespace, world_size: int) -> tupl def build_module_grid_specs( args: argparse.Namespace, world_size: int, encoder_module_name: str ) -> List[ModuleGridSpec]: - """Map grid args to the ModuleGridSpec list create_topology consumes; caller supplies encoder_module_name.""" + """Map grid args to the ModuleGridSpec list create_topology consumes.""" encoder_size, llm_size = validate_hetero_grid_args(args, world_size) language_grid_spec = ModuleGridSpec( @@ -136,8 +130,8 @@ def build_module_grid_specs( name=encoder_module_name, num_ranks=encoder_size, tp=args.encoder_tp, - cp=args.encoder_cp, - pp=args.encoder_pp, + cp=1, + pp=1, ep=1, rank_offset=0, expt_tp=1, diff --git a/examples/mimo/training/topology.py b/examples/mimo/training/topology.py index 7a79f0dcad6..b3fa4c94d14 100644 --- a/examples/mimo/training/topology.py +++ b/examples/mimo/training/topology.py @@ -195,7 +195,6 @@ def pg_collection_from_grid( pgc.dp_cp = grid.get_pg(["dp", "cp"]) pgc.intra_dp_cp = pgc.dp_cp pgc.tp_cp = grid.get_pg(["tp", "cp"]) - # MoE/finalize_model_grads read tp_dp_cp; cuda-graph capture reads tp_dp. pgc.tp_dp = grid.get_pg(["tp", "dp"]) pgc.tp_dp_cp = grid.get_pg(["tp", "dp", "cp"]) pgc.mp = grid.get_pg(["tp", "pp"]) diff --git a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py index fa6b6589e1f..7429e87434e 100644 --- a/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py +++ b/tests/unit_tests/models/mimo/test_mimo_hetero_grid_args.py @@ -26,9 +26,9 @@ def _parse(argv): def _layout_8gpu_20l(**overrides): - """Canonical 8-GPU layout: encoder 0-3 (tp2/pp1/dp2), llm 4-7 (tp2/pp1/dp2/ep4).""" + """Canonical 8-GPU layout: encoder 0-3 (tp2/dp2), llm 4-7 (tp2/pp1/dp2/ep4).""" argv = ( - "--encoder-tp 2 --encoder-pp 1 --encoder-dp 2 " + "--encoder-tp 2 --encoder-dp 2 " "--llm-offset 4 --llm-tp 2 --llm-pp 1 --llm-dp 2 --llm-ep 4" ).split() args = _parse(argv) @@ -51,7 +51,9 @@ def test_canonical_layout_validates_and_maps_specs(): assert encoder_grid_spec.name == "radio_encoder" assert encoder_grid_spec.num_ranks == 4 assert encoder_grid_spec.rank_offset == 0 # encoder span always starts at rank 0 - assert encoder_grid_spec.dp == 2 # derived: 4 // (tp2 * cp1 * pp1) + assert encoder_grid_spec.cp == 1 + assert encoder_grid_spec.pp == 1 + assert encoder_grid_spec.dp == 2 # derived: 4 // tp2 assert language_grid_spec.name == MIMO_LANGUAGE_MODULE_KEY assert language_grid_spec.num_ranks == 4 assert language_grid_spec.rank_offset == 4 @@ -88,7 +90,14 @@ def test_ep_divisibility_raises(): validate_hetero_grid_args(args, WORLD_SIZE_8) -def test_cp_must_be_one(): +def test_parser_does_not_expose_unsupported_grid_knobs(): + args = _parse([]) + assert not hasattr(args, "encoder_cp") + assert not hasattr(args, "encoder_pp") + assert not hasattr(args, "llm_expt_dp") + + +def test_llm_cp_must_be_one(): args = _layout_8gpu_20l(llm_cp=2) with pytest.raises(ValueError, match="CP=1 only"): validate_hetero_grid_args(args, WORLD_SIZE_8)