Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
301d12e
Add QK layernorm support for dot-product attention in MambaModel
Phlip79 Mar 31, 2026
367b8a8
Add tests for MambaModel QK layernorm spec functions
Phlip79 Mar 31, 2026
1c46827
Fix linting
Phlip79 Mar 31, 2026
b28a11f
Fix linting again
Phlip79 Mar 31, 2026
97607d6
Ran isort
Phlip79 Mar 31, 2026
bdf2d12
Update qk layernorm settings and tests
Phlip79 Mar 31, 2026
488e448
Fix linting
Phlip79 Mar 31, 2026
e9c20a4
Fix linting again
Phlip79 Mar 31, 2026
d16170f
Fix TENorm isinstance check in QK layernorm test
Phlip79 Mar 31, 2026
2ee490b
Use blueprint system instead
Phlip79 Apr 3, 2026
c778f9c
Update comments and failure
Phlip79 Apr 3, 2026
cb2c31e
Merge branch 'main' of https://github.com/NVIDIA/Megatron-LM into phi…
Phlip79 Apr 3, 2026
046f443
Update ordering
Phlip79 Apr 7, 2026
cd26bb4
Add TODO
Phlip79 Apr 10, 2026
d09c4ca
Refactor QK norm resolution: explicit branches + ValueError
Phlip79 Apr 16, 2026
861fe3b
Merge remote-tracking branch 'origin/philip/add-qk-norm' into philip/…
Phlip79 Apr 16, 2026
a49723f
Update test_l2_qk_norm to set qk_l2_norm on config
Phlip79 Apr 16, 2026
e16a762
Set qk_l2_norm on shared test config instead of duplicating
Phlip79 Apr 17, 2026
236c50f
Add tests for config-driven QK norm fallback and spec/config mismatch
Phlip79 Apr 17, 2026
c8a313d
Fix black formatting in test_qk_layernorm_spec_config_mismatch_raises
Phlip79 Apr 17, 2026
1ca9061
Fix TENorm isinstance check and add L2Norm config-fallback test
Phlip79 Apr 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions mamba_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
from megatron.core.transformer.spec_utils import import_module
from megatron.training import print_rank_0
from megatron.training.arguments import core_transformer_config_from_args
from megatron.core.models.mamba.mamba_layer_specs import mamba_inference_stack_spec
from megatron.core.models.mamba.mamba_layer_specs import (
get_mamba_inference_stack_spec,
get_mamba_stack_spec,
)


def mamba_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_collection=None):
Expand All @@ -16,14 +19,14 @@ def mamba_builder(args, pre_process, post_process, vp_stage=None, config=None, p
assert args.use_legacy_models is False, "Mamba only supported in Mcore!"

if config.transformer_impl == "inference_optimized":
mamba_stack_spec = mamba_inference_stack_spec
mamba_stack_spec = get_mamba_inference_stack_spec(config)
assert (
not config.inference_fuse_tp_communication
), "inference_fuse_tp_communication is not supported for Mamba"
elif args.spec is not None:
mamba_stack_spec = import_module(args.spec)
else:
raise ValueError("You must provide a valid Mamba layer spec via --spec")
mamba_stack_spec = get_mamba_stack_spec(config)

model = MambaModel(
config=config,
Expand Down
253 changes: 147 additions & 106 deletions megatron/core/models/mamba/mamba_layer_specs.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
)
from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules
from megatron.core.transformer.enums import AttnMaskType
from megatron.core.transformer.identity_op import IdentityOp
from megatron.core.transformer.mlp import MLP, MLPSubmodules
from megatron.core.transformer.multi_token_prediction import (
MultiTokenPredictionBlock,
Expand All @@ -31,6 +32,7 @@
MultiTokenPredictionLayerSubmodules,
)
from megatron.core.transformer.spec_utils import ModuleSpec
from megatron.core.transformer.torch_norm import L2Norm
from megatron.core.transformer.transformer_layer import (
MoETransformerLayer,
TransformerLayer,
Expand Down Expand Up @@ -69,136 +71,175 @@
)


mamba_stack_spec = ModuleSpec(
module=MambaStack,
submodules=MambaStackSubmodules(
mamba_layer=ModuleSpec(
module=MambaLayer,
submodules=MambaLayerSubmodules(
mixer=ModuleSpec(
module=MambaMixer,
submodules=MambaMixerSubmodules(
in_proj=TELayerNormColumnParallelLinear, out_proj=TERowParallelLinear
def _get_qk_layernorm(config):
"""Resolve q/k layernorm class from config."""
if config and config.qk_l2_norm:
return L2Norm
if config and config.qk_layernorm:
Comment thread
Phlip79 marked this conversation as resolved.
Resolved
return TENorm
return IdentityOp


def get_mamba_stack_spec(config=None):
"""Get the Mamba stack spec with Transformer Engine modules.

Args:
config: TransformerConfig. When provided, config-dependent features
(e.g. qk_layernorm) are applied. When None, defaults are used.
"""
qk_norm = _get_qk_layernorm(config)

return ModuleSpec(
module=MambaStack,
submodules=MambaStackSubmodules(
mamba_layer=ModuleSpec(
module=MambaLayer,
submodules=MambaLayerSubmodules(
mixer=ModuleSpec(
module=MambaMixer,
submodules=MambaMixerSubmodules(
in_proj=TELayerNormColumnParallelLinear, out_proj=TERowParallelLinear
),
),
mamba_bda=get_bias_dropout_add,
),
mamba_bda=get_bias_dropout_add,
),
),
gdn_layer=ModuleSpec(
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
self_attention=ModuleSpec(
module=GatedDeltaNet,
submodules=GatedDeltaNetSubmodules(
in_proj=TELayerNormColumnParallelLinear,
out_norm=TENorm,
out_proj=TERowParallelLinear,
gdn_layer=ModuleSpec(
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
self_attention=ModuleSpec(
module=GatedDeltaNet,
submodules=GatedDeltaNetSubmodules(
in_proj=TELayerNormColumnParallelLinear,
out_norm=TENorm,
out_proj=TERowParallelLinear,
),
),
self_attn_bda=get_bias_dropout_add,
),
self_attn_bda=get_bias_dropout_add,
),
),
# Started with spec from gpt_layer_specs.py (with MLP removed)
# Using the TE spec because we had problems getting the non-TE spec
# working
attention_layer=ModuleSpec(
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
self_attention=ModuleSpec(
module=SelfAttention,
params={"attn_mask_type": AttnMaskType.causal},
submodules=SelfAttentionSubmodules(
linear_qkv=TELayerNormColumnParallelLinear,
core_attention=TEDotProductAttention,
linear_proj=TERowParallelLinear,
# Started with spec from gpt_layer_specs.py (with MLP removed)
# Using the TE spec because we had problems getting the non-TE spec
# working
attention_layer=ModuleSpec(
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
self_attention=ModuleSpec(
module=SelfAttention,
params={"attn_mask_type": AttnMaskType.causal},
submodules=SelfAttentionSubmodules(
linear_qkv=TELayerNormColumnParallelLinear,
core_attention=TEDotProductAttention,
linear_proj=TERowParallelLinear,
q_layernorm=qk_norm,
k_layernorm=qk_norm,
),
),
self_attn_bda=get_bias_dropout_add,
),
self_attn_bda=get_bias_dropout_add,
),
),
# Started with spec from gpt_layer_specs.py
# Using the TE spec because we had problems getting the non-TE spec
# working
mlp_layer=ModuleSpec(
module=MLPLayer,
submodules=TransformerLayerSubmodules(
mlp=ModuleSpec(
module=MLP,
submodules=MLPSubmodules(
linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear
# Started with spec from gpt_layer_specs.py
# Using the TE spec because we had problems getting the non-TE spec
# working
mlp_layer=ModuleSpec(
module=MLPLayer,
submodules=TransformerLayerSubmodules(
mlp=ModuleSpec(
module=MLP,
submodules=MLPSubmodules(
linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear
),
),
mlp_bda=get_bias_dropout_add,
),
mlp_bda=get_bias_dropout_add,
),
),
moe_layer=ModuleSpec(
module=MoETransformerLayer,
submodules=TransformerLayerSubmodules(
pre_mlp_layernorm=TENorm, mlp=moe, mlp_bda=get_bias_dropout_add
moe_layer=ModuleSpec(
module=MoETransformerLayer,
submodules=TransformerLayerSubmodules(
pre_mlp_layernorm=TENorm, mlp=moe, mlp_bda=get_bias_dropout_add
),
),
mtp_block_spec=_mamba_mtp_block_spec,
),
mtp_block_spec=_mamba_mtp_block_spec,
),
)
)


# Backward-compatible constant (no QK norm).
mamba_stack_spec = get_mamba_stack_spec()

mamba_inference_stack_spec = ModuleSpec(
module=MambaStack,
submodules=MambaStackSubmodules(
mamba_layer=ModuleSpec(
module=MambaLayer,
submodules=MambaLayerSubmodules(
mixer=ModuleSpec(
module=MambaMixer,
submodules=MambaMixerSubmodules(
in_proj=InferenceLayerNormColumnParallelLinear,
out_proj=InferenceRowParallelLinear,

def get_mamba_inference_stack_spec(config=None):
"""Get the inference-optimized Mamba stack spec.

Args:
config: TransformerConfig. When provided, config-dependent features
(e.g. qk_layernorm) are applied. When None, defaults are used.
"""
qk_norm = _get_qk_layernorm(config)

return ModuleSpec(
module=MambaStack,
submodules=MambaStackSubmodules(
mamba_layer=ModuleSpec(
module=MambaLayer,
submodules=MambaLayerSubmodules(
mixer=ModuleSpec(
module=MambaMixer,
submodules=MambaMixerSubmodules(
in_proj=InferenceLayerNormColumnParallelLinear,
out_proj=InferenceRowParallelLinear,
),
),
mamba_bda=get_bias_dropout_add,
),
mamba_bda=get_bias_dropout_add,
),
),
# Started with spec from gpt_layer_specs.py (with MLP removed)
# Using the TE spec because we had problems getting the non-TE spec
# working
attention_layer=ModuleSpec(
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
self_attention=ModuleSpec(
module=SelfAttention,
params={"attn_mask_type": AttnMaskType.causal},
submodules=SelfAttentionSubmodules(
linear_qkv=InferenceLayerNormColumnParallelLinear,
core_attention=TEDotProductAttention,
linear_proj=InferenceRowParallelLinear,
# Started with spec from gpt_layer_specs.py (with MLP removed)
# Using the TE spec because we had problems getting the non-TE spec
# working
attention_layer=ModuleSpec(
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
self_attention=ModuleSpec(
module=SelfAttention,
params={"attn_mask_type": AttnMaskType.causal},
submodules=SelfAttentionSubmodules(
linear_qkv=InferenceLayerNormColumnParallelLinear,
core_attention=TEDotProductAttention,
linear_proj=InferenceRowParallelLinear,
q_layernorm=qk_norm,
k_layernorm=qk_norm,
),
),
self_attn_bda=get_bias_dropout_add,
),
self_attn_bda=get_bias_dropout_add,
),
),
# Started with spec from gpt_layer_specs.py
# Using the TE spec because we had problems getting the non-TE spec
# working
mlp_layer=ModuleSpec(
module=MLPLayer,
submodules=TransformerLayerSubmodules(
mlp=ModuleSpec(
module=MLP,
submodules=MLPSubmodules(
linear_fc1=InferenceLayerNormColumnParallelLinear,
linear_fc2=InferenceRowParallelLinear,
# Started with spec from gpt_layer_specs.py
# Using the TE spec because we had problems getting the non-TE spec
# working
mlp_layer=ModuleSpec(
module=MLPLayer,
submodules=TransformerLayerSubmodules(
mlp=ModuleSpec(
module=MLP,
submodules=MLPSubmodules(
linear_fc1=InferenceLayerNormColumnParallelLinear,
linear_fc2=InferenceRowParallelLinear,
),
),
mlp_bda=get_bias_dropout_add,
),
mlp_bda=get_bias_dropout_add,
),
),
moe_layer=ModuleSpec(
# Use inference-optimized MoE layer for end-to-end CUDA graph support
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
pre_mlp_layernorm=TENorm, mlp=moe_inference, mlp_bda=get_bias_dropout_add
moe_layer=ModuleSpec(
# Use inference-optimized MoE layer for end-to-end CUDA graph support
module=TransformerLayer,
submodules=TransformerLayerSubmodules(
pre_mlp_layernorm=TENorm, mlp=moe_inference, mlp_bda=get_bias_dropout_add
),
),
mtp_block_spec=_mamba_mtp_block_spec,
),
mtp_block_spec=_mamba_mtp_block_spec,
),
)
)


# Backward-compatible constant (no QK norm).
mamba_inference_stack_spec = get_mamba_inference_stack_spec()