Skip to content
Merged
5 changes: 3 additions & 2 deletions megatron/core/distributed/param_and_grad_buffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -840,8 +840,9 @@ def group_params_for_buffers(
Each distinct buffer is identified by a BufferKey with three dimensions:
- param_dtype: storage dtype (torch.uint8 for FP8/NVFP4 parameters, else param.dtype).
- grad_dtype: gradient reduction dtype (torch.float if grad_reduce_in_fp32, else param.dtype).
- is_expert_parallel: whether the parameter is expert-parallel (param.allreduce == False),
which requires a separate buffer with a different data-parallel group.
- is_expert_parallel: whether the parameter uses the expert topology (param.allreduce == False),
which requires a separate buffer for the expert data-parallel group. This is true for experts
when expert-parallelism > 1 or expert-tensor-parallelism != tensor-parallelism.

The param_indices track each parameter's position among same-dtype params (using
the "fake" high-precision dtype for FP8/NVFP4 params), needed for loading non-native-fp8
Expand Down
86 changes: 64 additions & 22 deletions megatron/core/extensions/transformer_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import io
import os
import pickle
import re
import warnings
from contextlib import nullcontext
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, cast
Expand Down Expand Up @@ -84,6 +85,42 @@
HAVE_TE = False

_TE_CONFIG_TYPE_KEY = "transformer_engine_config_type"
_EXPERT_PARAMETER_NAME_PATTERN = re.compile(r"(weight|bias)\d*")


def _set_expert_parameter_attributes(
module: torch.nn.Module, parallel_mode: Optional[str], use_expert_pgs: bool
) -> None:
"""Set process-group and tensor-partition metadata on an expert TE module.

``allreduce=False`` selects EDP for gradient reduction.

Weights and biases, including TEGroupedLinear's numbered parameters, are also marked as
TP-partitioned according to ``parallel_mode``; row-parallel biases remain replicated.

Any parameter which is partitioned along TP or ETP is marked with ``tensor_model_parallel``,
which ensures that all shards contribute to the gradient norm.

Args:
module: Transformer Engine module whose direct parameters should be marked.
parallel_mode: Tensor-parallel mode used by the module (``"column"``, ``"row"``, or None).
use_expert_pgs: Whether to use EP/ETP/EDP process groups instead of TP/CP/DP.
"""
for name, param in module.named_parameters(recurse=False):
Comment thread
philipcmonk marked this conversation as resolved.
Resolved
param.allreduce = not use_expert_pgs

name_match = _EXPERT_PARAMETER_NAME_PATTERN.fullmatch(name)
parameter_kind = name_match.group(1) if name_match else None
is_weight = parameter_kind == "weight"
is_bias = parameter_kind == "bias"
is_partitioned = parallel_mode in ("column", "row") and (
is_weight or (parallel_mode == "column" and is_bias)
)
if is_weight or is_bias:
param.tensor_model_parallel = is_partitioned
if is_partitioned:
param.partition_dim = 1 if parallel_mode == "row" else 0
param.partition_stride = 1


class TransformerEngineConfigType(enum.Enum):
Expand Down Expand Up @@ -857,6 +894,10 @@ def __init__(
tp_size = get_pg_size(tp_group)

self.expert_parallel = self.config.expert_model_parallel_size > 1
use_expert_pgs = is_expert and (
self.expert_parallel
or self.config.expert_tensor_parallel_size != self.config.tensor_model_parallel_size
)
if is_expert:
rng_tracker_name = get_expert_parallel_rng_tracker_name()
else:
Expand Down Expand Up @@ -914,11 +955,10 @@ def __init__(
**extra_kwargs,
)

for param in self.parameters():
if is_expert:
# Reduce the gradient on the expert_data_parallel group for expert linear layers
setattr(param, "allreduce", not self.expert_parallel)
else:
if is_expert:
_set_expert_parameter_attributes(self, parallel_mode, use_expert_pgs)
else:
for param in self.parameters():
# Reduce the gradient on DP group
setattr(param, "allreduce", True)
if parallel_mode == "duplicated":
Expand Down Expand Up @@ -1313,6 +1353,13 @@ def __init__(
self.bias.zero_()
setattr(self.bias, "allreduce", True)

if is_expert:
use_expert_pgs = (
config.expert_model_parallel_size > 1
or config.expert_tensor_parallel_size != config.tensor_model_parallel_size
)
_set_expert_parameter_attributes(self, "column", use_expert_pgs)

def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
"""Sharding along axis 0, bias sharded"""
state_dict = self.state_dict(prefix="", keep_vars=True)
Expand Down Expand Up @@ -1552,6 +1599,13 @@ def __init__(
setattr(self.bias, "allreduce", True)
setattr(self.bias, "sequence_parallel", config.sequence_parallel)

if is_expert:
use_expert_pgs = (
config.expert_model_parallel_size > 1
or config.expert_tensor_parallel_size != config.tensor_model_parallel_size
)
_set_expert_parameter_attributes(self, "row", use_expert_pgs)

def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
"""Sharding along axis 1, bias not sharded"""
state_dict = self.state_dict(prefix="", keep_vars=True)
Expand Down Expand Up @@ -1970,6 +2024,10 @@ def __init__(
extra_kwargs["ub_name"] = tp_comm_buffer_name

self.expert_parallel = self.config.expert_model_parallel_size > 1
use_expert_pgs = is_expert and (
self.expert_parallel
or self.config.expert_tensor_parallel_size != self.config.tensor_model_parallel_size
)
if is_expert:
extra_kwargs["rng_tracker_name"] = get_expert_parallel_rng_tracker_name()

Expand Down Expand Up @@ -2036,23 +2094,7 @@ def __init__(
**extra_kwargs,
)

for param in self.parameters():
setattr(param, "allreduce", not (is_expert and self.expert_parallel))

# Explicitly stamp partition_dim and partition_stride on expert weight
# tensors when explicit_expert_comm cleared parallel_mode. TE ≤2.12
# set these internally; TE ≥2.13 no longer does (parallel_mode=None
# is passed due to explicit_expert_comm). The resharding/refit planner
# relies on partition_dim to correctly plan TP gather/scatter operations.
# NOTE: we intentionally do NOT stamp tensor_model_parallel here —
# doing so would change num-zeros gradient counting.
if self.explicit_expert_comm and original_parallel_mode in ("column", "row"):
part_dim = 0 if original_parallel_mode == "column" else 1
for i in range(num_gemms):
weight = getattr(self, f"weight{i}", None)
if weight is not None:
setattr(weight, "partition_dim", part_dim)
setattr(weight, "partition_stride", 1)
_set_expert_parameter_attributes(self, original_parallel_mode, use_expert_pgs)

self._register_load_state_dict_pre_hook(
type(self)._normalize_grouped_parameter_keys, with_module=True
Expand Down
17 changes: 8 additions & 9 deletions megatron/core/optimizer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
if HAVE_EMERGING_OPTIMIZERS:
from emerging_optimizers.scalar_optimizers import Lion

from megatron.core import parallel_state
from megatron.core.optimizer.cpu_offloading.hybrid_optimizer import HybridDeviceOptimizer
from megatron.core.optimizer_param_scheduler import (
ParamGroupOverride,
Expand Down Expand Up @@ -686,11 +685,12 @@ def init_state_fn(opt, config=None):
setattr(optimizer, 'grad_stats_parallel_group', model_parallel_group)

if pg_collection is None or not hasattr(pg_collection, 'tp'):
tp_group = parallel_state.get_tensor_model_parallel_group()
else:
tp_group = pg_collection.tp
# TODO(M4): plumb tp_group through optimizer constructors so this setattr disappears.
pg_collection = ProcessGroupCollection.use_mpu_process_groups()
Comment thread
deepakn94 marked this conversation as resolved.
Resolved
tp_group = pg_collection.tp
expert_tp_group = getattr(pg_collection, 'expt_tp', tp_group)
# TODO(M4): plumb TP groups through optimizer constructors so these setattrs disappear.
setattr(optimizer, 'tp_group', tp_group)
setattr(optimizer, 'expert_tp_group', expert_tp_group)

return optimizer

Expand Down Expand Up @@ -884,11 +884,10 @@ def _get_megatron_emerging_optimizer(
else:
optimizer = FP32Optimizer(optimizer, config, init_state_fn)
setattr(optimizer, 'grad_stats_parallel_group', model_parallel_group)
if pg_collection is None or not hasattr(pg_collection, 'tp'):
tp_group = parallel_state.get_tensor_model_parallel_group()
else:
tp_group = pg_collection.tp
tp_group = pg_collection.tp
expert_tp_group = getattr(pg_collection, 'expt_tp', tp_group)
setattr(optimizer, 'tp_group', tp_group)
setattr(optimizer, 'expert_tp_group', expert_tp_group)
results.append(optimizer)
continue
else:
Expand Down
5 changes: 4 additions & 1 deletion megatron/core/optimizer/clip_grads.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ def count_zeros_fp32(
grad_stats_parallel_group: torch.distributed.ProcessGroup,
use_decoupled_grad: bool = False,
tp_group: Optional[torch.distributed.ProcessGroup] = None,
expert_tp_group: Optional[torch.distributed.ProcessGroup] = None,
) -> float:
"""Counts the number of zero values in the gradients of the given parameters.

Expand Down Expand Up @@ -241,7 +242,9 @@ def count_zeros_fp32(
total_num_zeros += num_zeros
continue
is_not_shared = param_is_not_shared(param)
is_not_tp_duplicate = param_is_not_tensor_parallel_duplicate(param, tp_group=tp_group)
is_not_tp_duplicate = param_is_not_tensor_parallel_duplicate(
param, tp_group=tp_group, expert_tp_group=expert_tp_group
)
if grad_not_none and is_not_shared and is_not_tp_duplicate:
grad_obj = getattr(param, grad_attr)
data_parallel_group = get_data_parallel_group_if_dtensor(grad_obj, data_parallel_group)
Expand Down
9 changes: 8 additions & 1 deletion megatron/core/optimizer/optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ def _is_separate_grad_norm_group(grad_norm_group: Optional[str]) -> bool:

def copy_optimizer_param_metadata(destination: torch.Tensor, source: torch.Tensor) -> None:
"""Copy optimizer-relevant metadata when creating param views/copies."""
if hasattr(source, 'allreduce'):
destination.allreduce = source.allreduce
if hasattr(source, 'shared'):
destination.shared = source.shared
if hasattr(source, GRAD_NORM_GROUP_ATTR):
Expand Down Expand Up @@ -213,7 +215,9 @@ def _filter_grads_for_norm(
grad_not_none = grad is not None
is_not_shared = param_is_not_shared(param)
is_not_tp_duplicate = tensor_parallel.param_is_not_tensor_parallel_duplicate(
param, getattr(self, 'tp_group', None)
param,
tp_group=getattr(self, 'tp_group', None),
expert_tp_group=getattr(self, 'expert_tp_group', None),
)
if grad_not_none and is_not_shared and is_not_tp_duplicate:
grads_for_norm.append(grad)
Expand Down Expand Up @@ -380,6 +384,7 @@ def count_zeros(self) -> float:
and getattr(params[0], "__fsdp_param__", False)
),
tp_group=getattr(self, 'tp_group', None),
expert_tp_group=getattr(self, 'expert_tp_group', None),
)

@abstractmethod
Expand Down Expand Up @@ -1600,6 +1605,8 @@ def count_zeros(self):
self.config.use_precision_aware_optimizer
and getattr(params[0], "__fsdp_param__", False)
),
tp_group=getattr(self.chained_optimizers[0], 'tp_group', None),
expert_tp_group=getattr(self.chained_optimizers[0], 'expert_tp_group', None),
)
else:
num_zeros_in_grad = 0
Expand Down
7 changes: 6 additions & 1 deletion megatron/core/post_training/modelopt/layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,12 @@ def __init__(
for param in self.parameters():
if is_expert:
# Reduce the gradient on the expert_data_parallel group for expert linear layers
setattr(param, "allreduce", self.config.expert_model_parallel_size == 1)
use_expert_groups = (
self.config.expert_model_parallel_size > 1
or self.config.expert_tensor_parallel_size
!= self.config.tensor_model_parallel_size
)
setattr(param, "allreduce", not use_expert_groups)
else:
# Reduce the gradient on DP group
setattr(param, "allreduce", True)
Expand Down
27 changes: 20 additions & 7 deletions megatron/core/tensor_parallel/layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,16 @@
dist_reduce_scatter_func = torch.distributed._reduce_scatter_base


def param_is_not_tensor_parallel_duplicate(param, tp_group=None):
"""Returns true if the passed-in parameter is not a duplicate parameter
on another TP rank."""
def param_is_not_tensor_parallel_duplicate(param, tp_group=None, expert_tp_group=None):
"""Return whether a parameter contributes a unique model-parallel shard.
Comment thread
philipcmonk marked this conversation as resolved.
Resolved

Parameters reduced over expert data parallel groups use the expert tensor-parallel
group for duplicate filtering. Other parameters use the regular tensor-parallel group.
"""
if hasattr(param, "tensor_model_parallel") and param.tensor_model_parallel:
return True
if not getattr(param, "allreduce", True) and expert_tp_group is not None:
Comment thread
philipcmonk marked this conversation as resolved.
Resolved
tp_group = expert_tp_group
# Prefer provided tp_group when available (new explicit path).
if tp_group is not None:
return tp_group.rank() == 0
Expand Down Expand Up @@ -867,6 +872,10 @@ def __init__(
world_size = get_pg_size(self.tp_group)
rank = get_pg_rank(self.tp_group)
self.explicit_expert_comm = self.is_expert and (world_size > 1 or self.expert_parallel)
use_expert_groups = self.is_expert and (
self.expert_parallel
or self.config.expert_tensor_parallel_size != self.config.tensor_model_parallel_size
)
self.output_size_per_partition = divide(output_size, world_size)

# Parameters.
Expand Down Expand Up @@ -919,7 +928,7 @@ def __init__(
tensor=self.weight, is_parallel=True, dim=0, stride=stride
)

setattr(self.weight, "allreduce", not (self.is_expert and self.expert_parallel))
setattr(self.weight, "allreduce", not use_expert_groups)
else:
self.weight = None

Expand All @@ -941,7 +950,7 @@ def __init__(
# Always initialize bias to zero.
with torch.no_grad():
self.bias.zero_()
setattr(self.bias, "allreduce", not (self.is_expert and self.expert_parallel))
setattr(self.bias, "allreduce", not use_expert_groups)
else:
self.register_parameter("bias", None)

Expand Down Expand Up @@ -1269,7 +1278,11 @@ def __init__(
set_tensor_model_parallel_attributes(
tensor=self.weight, is_parallel=True, dim=1, stride=stride
)
setattr(self.weight, "allreduce", not (self.is_expert and self.expert_parallel))
use_expert_groups = self.is_expert and (
Comment thread
philipcmonk marked this conversation as resolved.
Resolved
self.expert_parallel
or self.config.expert_tensor_parallel_size != self.config.tensor_model_parallel_size
)
setattr(self.weight, "allreduce", not use_expert_groups)

if bias:
if config.use_cpu_initialization:
Expand All @@ -1287,7 +1300,7 @@ def __init__(
# Always initialize bias to zero.
with torch.no_grad():
self.bias.zero_()
setattr(self.bias, "allreduce", not (self.is_expert and self.expert_parallel))
setattr(self.bias, "allreduce", not use_expert_groups)
setattr(self.bias, "sequence_parallel", self.sequence_parallel)
else:
self.register_parameter("bias", None)
Expand Down
31 changes: 28 additions & 3 deletions megatron/training/utils/common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,23 +74,29 @@ def calc_params_l2_norm(model, force_create_fp32_copy=False):
params_data = []
moe_params_data = []
sharded_params_data = []
sharded_moe_params_data = []
data_parallel_group = None
tp_group = mpu.get_tensor_model_parallel_group()
expert_tp_group = mpu.get_expert_tensor_parallel_group()

for model_chunk in model:
for param in model_chunk.parameters():
data_parallel_group = get_data_parallel_group_if_dtensor(param, data_parallel_group)
is_not_tp_duplicate = param_is_not_tensor_parallel_duplicate(param)
is_expert = not getattr(param, 'allreduce', True)
is_not_tp_duplicate = param_is_not_tensor_parallel_duplicate(
param, tp_group=tp_group, expert_tp_group=expert_tp_group
)
if not is_not_tp_duplicate:
continue
assert is_not_tp_duplicate
if not getattr(param, 'allreduce', True):
if is_expert:
assert param_is_not_shared(param)
param = to_local_if_dtensor(param)
if args.bf16:
if not force_create_fp32_copy and hasattr(param, 'main_param'):
if getattr(param, 'main_param_sharded', False):
if param.main_param is not None:
sharded_params_data.append(param.main_param)
sharded_moe_params_data.append(param.main_param)
else:
moe_params_data.append(param.main_param)
else:
Expand Down Expand Up @@ -170,6 +176,25 @@ def calc_params_l2_norm(model, force_create_fp32_copy=False):
else:
moe_norm_2 = torch.zeros_like(norm_2)

# Distributed optimizer shards expert main parameters over expert DP rather than
# regular DP, so reduce their contribution separately before the model-parallel sum.
if len(sharded_moe_params_data) > 0:
sharded_moe_norm, _ = multi_tensor_applier(
multi_tensor_l2norm,
dummy_overflow_buf,
[sharded_moe_params_data],
False, # no per-parameter norm.
)
sharded_moe_norm_2 = sharded_moe_norm * sharded_moe_norm
else:
sharded_moe_norm_2 = torch.zeros((1,), dtype=torch.float32, device='cuda')
torch.distributed.all_reduce(
sharded_moe_norm_2,
op=torch.distributed.ReduceOp.SUM,
group=mpu.get_expert_data_parallel_group(),
)
moe_norm_2 += sharded_moe_norm_2

# Reduce norm across model parallel groups (dense and expert).
# Dense params should sum across all model-parallel GPUs (tensor + pipeline).
dense_reduce_group = mpu.get_model_parallel_group()
Expand Down
Loading
Loading