Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion megatron/core/extensions/transformer_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -1708,11 +1708,11 @@ def __init__(
self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache

extra_kwargs = _get_extra_te_kwargs(config)

self.delay_wgrad_compute = (
self.config.delay_wgrad_compute
or self.config.overlap_dispatch_backward_with_experts_wgrad
)

if self.delay_wgrad_compute:
if is_te_min_version("2.3.0"):
extra_kwargs["delay_wgrad_compute"] = True
Expand Down Expand Up @@ -1756,6 +1756,14 @@ def __init__(
tp_size = 1
tp_group_for_te = None

if is_te_min_version("2.14.0"):
extra_kwargs["single_grouped_weight"] = getattr(
config, "moe_single_grouped_weight", False
)
extra_kwargs["single_grouped_bias"] = getattr(
config, "moe_single_grouped_bias", False
)

super().__init__(
num_gemms=num_gemms,
in_features=input_size,
Expand Down
39 changes: 31 additions & 8 deletions megatron/core/transformer/moe/experts.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,11 @@
from megatron.core.utils import is_te_min_version

if HAVE_TE:
import transformer_engine as te

from megatron.core.extensions.transformer_engine import Fp8Padding, Fp8Unpadding
else:
te = None # type: ignore[assignment, misc]
Fp8Padding, Fp8Unpadding = None, None

try:
Expand Down Expand Up @@ -790,11 +793,18 @@ def _is_fused_impl_supported(self) -> bool:
if not isinstance(self.linear_fc2, te.pytorch.GroupedLinear):
return False

# Check activation
if self.activation_func != F.silu or not self.config.gated_linear_unit:
return False # Expected SwiGLU activation
# Check activation: SwiGLU (ScaledSwiGLU) or quick GEGLU (ScaledClampedQGeGLU, TE >= 2.15)
if self.config.gated_linear_unit:
if self.activation_func == F.silu:
return True
if self.activation_func == quick_gelu:
try:
from transformer_engine.pytorch.ops import ScaledClampedQGeGLU # noqa: F401
except ImportError:
return False
return True

return True
return False

def _make_fused_ops(self) -> torch.nn.Module:
"""Construct fused module for FC1, activation, and FC2."""
Expand Down Expand Up @@ -845,10 +855,23 @@ def _make_fused_ops(self) -> torch.nn.Module:
setattr(op, "bias", getattr(self.linear_fc1, "bias"))
ops.append(op)

# Activation and post-multiply probs
op = te.pytorch.ops.ScaledSwiGLU(
glu_interleave_size=self.config.moe_mlp_glu_interleave_size
)
# Activation and post-multiply probs (SwiGLU or clamped quick-GEGL)
glu_interleave = self.config.moe_mlp_glu_interleave_size
if self.activation_func == F.silu and self.config.gated_linear_unit:
op = te.pytorch.ops.ScaledSwiGLU(glu_interleave_size=glu_interleave)
elif self.activation_func == quick_gelu and self.config.gated_linear_unit:
clamp = self.config.activation_func_clamp_value
if clamp is not None:
op = te.pytorch.ops.ScaledClampedQGeGLU(
glu_interleave_size=glu_interleave, limit=clamp
)
else:
op = te.pytorch.ops.ScaledClampedQGeGLU(glu_interleave_size=glu_interleave)
else:
raise RuntimeError(
"_make_fused_ops expected SwiGLU or quick_gelu with gated_linear_unit; "
"call _is_fused_impl_supported() before constructing fused ops."
)
ops.append(op)

# FC2
Expand Down
24 changes: 24 additions & 0 deletions megatron/core/transformer/transformer_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,16 @@ class TransformerConfig(ModelParallelConfig):
GEMM feature introduced since CUTLASS 2.8 (https://github.com/fanshiqing/grouped_gemm).
"""

moe_single_grouped_weight: bool = False
"""When using TE GroupedLinear for MoE experts, store expert weights as a single grouped
parameter via Transformer Engine's `GroupedTensor`. Requires ``moe_grouped_gemm=True``.
"""

moe_single_grouped_bias: bool = False
"""When using TE GroupedLinear for MoE experts, store expert biases as a single grouped
parameter via Transformer Engine's `GroupedTensor`. Requires ``moe_grouped_gemm=True``
and ``add_bias_linear=True``."""

moe_aux_loss_coeff: Union[float, List[float]] = 0.0
"""Scaling coefficient for the aux loss. A starting value of 1e-2 is recommended.
If a list of load balancing types is provided for `moe_router_load_balancing_type`,
Expand Down Expand Up @@ -1309,6 +1319,20 @@ def __post_init__(self):
self.moe_ffn_hidden_size is None
), "moe_ffn_hidden_size must be None when num_experts is not set."

if self.moe_single_grouped_weight or self.moe_single_grouped_bias:
if not self.moe_grouped_gemm:
raise ValueError(
"moe_single_grouped_weight and moe_single_grouped_bias require "
"moe_grouped_gemm=True."
)
if not is_te_min_version("2.14.0"):
raise ValueError(
"moe_single_grouped_weight and moe_single_grouped_bias require "
f"transformer-engine>=2.14.0, but your version is {get_te_version()}."
)
if self.moe_single_grouped_bias and not self.add_bias_linear:
raise ValueError("moe_single_grouped_bias requires add_bias_linear=True.")

if self.moe_enable_deepep:
if self.moe_token_dispatcher_type != "flex":
raise ValueError("DeepEP backend is only supported with flex token dispatcher.")
Expand Down
2 changes: 2 additions & 0 deletions megatron/training/checkpointing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1509,6 +1509,8 @@ def _set_arg(arg_name, old_arg_name=None, force=False):
_set_arg('moe_token_dispatcher_type', force=False)
_set_arg('moe_router_pre_softmax', force=True)
_set_arg('moe_grouped_gemm', force=True)
_set_arg('moe_single_grouped_weight', force=True)
_set_arg('moe_single_grouped_bias', force=True)
_set_arg('moe_shared_expert_intermediate_size', force=True)
_set_arg('moe_router_score_function', force=True)
_set_arg('moe_router_enable_expert_bias', force=True)
Expand Down
2 changes: 2 additions & 0 deletions tests/unit_tests/models/test_mamba_moe_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,8 @@
"moe_router_force_biased": None,
"sequence_packing_scheduler": None,
"use_transformer_engine_op_fuser": False,
"moe_single_grouped_weight": False,
"moe_single_grouped_bias": False,
}
# Fields to ignore entirely (ephemeral, environment-specific, very large).
SKIP_FIELDS = set()
Expand Down
Loading