Skip to content
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
33f0ec0
Adding Qwen3-Next model provider
adityavavreNVDA Oct 21, 2025
64ace9f
Fixing class inheritance
adityavavreNVDA Oct 21, 2025
66bacdd
Fixing router fw pass by adding pre-softmax and also enabling normali…
adityavavreNVDA Oct 21, 2025
6b648f0
Renaming and removing pre softmax + norm since it is equivalent to po…
adityavavreNVDA Oct 22, 2025
348eec5
Adding template for pretraining recipe
adityavavreNVDA Oct 22, 2025
1bcf697
Separating perf optimization flags from model provider
adityavavreNVDA Oct 22, 2025
3d76f6f
Renaming and adding functionality to add weight decayto qk layernorm
adityavavreNVDA Oct 22, 2025
88357a9
Removing bridge
adityavavreNVDA Oct 22, 2025
bfd1be6
Some renaming and doc string changes
adityavavreNVDA Oct 22, 2025
d5c0185
Fixing checkpointing issue and also changing the recommended parallel…
adityavavreNVDA Oct 25, 2025
907bf70
Fixing dependencies
adityavavreNVDA Oct 25, 2025
f2ded93
Fixing checkpointing issue
adityavavreNVDA Oct 27, 2025
24efa78
Removing extra newline
adityavavreNVDA Oct 27, 2025
62e4946
Adding model provider unit tests
adityavavreNVDA Oct 28, 2025
35afbbf
Fixing merge conflicts
adityavavreNVDA Oct 28, 2025
eded983
Chaning model provider to use bridge and use global_aux_loss for load…
adityavavreNVDA Oct 29, 2025
9a2910c
Add moe_flex_dispatcher_backend in model cfg for DEEPEP
adityavavreNVDA Oct 30, 2025
9b4c464
Adding qwen3-next docs
adityavavreNVDA Oct 30, 2025
e7f5c6d
Fixing docs issue
adityavavreNVDA Oct 31, 2025
0b5c5c4
Merge branch 'main' into avavre/qwen3next_support
adityavavreNVDA Oct 31, 2025
3f5a714
Fixing lint issues
adityavavreNVDA Nov 1, 2025
aabbceb
More lint fixes
adityavavreNVDA Nov 2, 2025
1459fcb
More lint fixes
adityavavreNVDA Nov 2, 2025
af21702
MTP test fix
adityavavreNVDA Nov 2, 2025
8bf18c2
MTP test fix
adityavavreNVDA Nov 2, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ classifiers = [
"Topic :: Utilities",
]
dependencies = [
"transformers==4.57.1",
"datasets",
"omegaconf>=2.3.0",
"tensorboard>=2.19.0",
Expand Down Expand Up @@ -94,6 +95,7 @@ no-build-isolation-package = [
"causal-conv1d",
"nv-grouped-gemm",
"flash_mla",
"flash-linear-attention",
]
prerelease = "allow"
override-dependencies = [
Expand Down
4 changes: 4 additions & 0 deletions src/megatron/bridge/models/qwen/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
Qwen3MoEModelProvider,
Qwen3MoEModelProvider30B_A3B,
Qwen3MoEModelProvider235B_A22B,
Qwen3NextModelProvider,
Qwen3NextModelProvider80B_A3B,
Qwen25ModelProvider1P5B,
Qwen25ModelProvider3B,
Qwen25ModelProvider7B,
Expand Down Expand Up @@ -64,4 +66,6 @@
"Qwen3MoEModelProvider",
"Qwen3MoEModelProvider30B_A3B",
"Qwen3MoEModelProvider235B_A22B",
"Qwen3NextModelProvider",
"Qwen3NextModelProvider80B_A3B",
]
66 changes: 65 additions & 1 deletion src/megatron/bridge/models/qwen/qwen_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,22 @@

import logging
from dataclasses import dataclass
from functools import partial
from typing import Callable, Optional

import torch
import torch.nn.functional as F

from megatron.bridge.models.gpt_provider import GPTModelProvider

from megatron.core.transformer.spec_utils import ModuleSpec
from megatron.core.models.gpt.gpt_layer_specs import get_gpt_decoder_block_spec

try:
import transformer_engine # type: ignore # noqa: F401

HAVE_TE = True
except (ImportError, ModuleNotFoundError):
HAVE_TE = False

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -401,3 +410,58 @@ class Qwen3MoEModelProvider235B_A22B(Qwen3MoEModelProvider):
num_query_groups: int = 4
ffn_hidden_size: int = 12288
moe_ffn_hidden_size: int = 1536

# =============================================================================
# Qwen 3-Next MoE Model Provider (based on GPTProvider)
# =============================================================================


@dataclass
class Qwen3NextModelProvider(Qwen3MoEModelProvider):
"""Base provider for Qwen 3 MoE Models."""

transformer_layer_spec: ModuleSpec | Callable[["GPTModelProvider"], ModuleSpec] = partial(
get_gpt_decoder_block_spec, use_transformer_engine=HAVE_TE
)

layernorm_zero_centered_gamma: bool = True # Zero-centered RMSNorm
kv_channels: int | None = 256
num_query_groups: int = 2
seq_length: int = 262144 # 256k tokens
rotary_base: float = 10000000.0
rotary_percent: float = 0.25 # 25% of the hidden size is used for RoPE
attention_output_gate: bool = True # Gated Attention

# MoE specific parameters
num_moe_experts: int = 512
moe_router_topk: int = 10 # 10 routed experts per token
moe_shared_expert_gate: bool = True # Qwen3-Next uses a gate for the shared expert
moe_router_dtype: str = "fp32"

# Linear Attention specific parameters
linear_attention_type: str = "gated_delta_net" # Gated Delta Net used in 75% of the model layers
linear_attention_freq: int | list[int] = 4 # 1 gated standard attention layer per 4 layers
linear_conv_kernel_dim: int = 4
linear_key_head_dim: int = 128
linear_value_head_dim: int = 128
linear_num_key_heads: int = 16
linear_num_value_heads: int = 32

# Checkpointing
hetereogenous_dist_checkpoint: bool = True

@dataclass
class Qwen3NextModelProvider80B_A3B(Qwen3NextModelProvider):
"""
Provider for Qwen 3 Next 80B-A3B: https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct and https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking
"""

num_layers: int = 48
hidden_size: int = 2048
num_attention_heads: int = 16
num_query_groups: int = 2
ffn_hidden_size: int = 5120
moe_ffn_hidden_size: int = 512
moe_shared_expert_intermediate_size: int = 512
mtp_num_layers: Optional[int] = None
mtp_loss_scaling_factor: Optional[float] = None
7 changes: 7 additions & 0 deletions src/megatron/bridge/recipes/qwen/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@
qwen3_235b_a22b_pretrain_config,
)

# Qwen3-Next models
from .qwen3_next import (
qwen3_next_80b_a3b_pretrain_config,
)


__all__ = [
# Qwen2 models
Expand All @@ -67,4 +72,6 @@
# Qwen3 MoE models
"qwen3_30b_a3b_pretrain_config",
"qwen3_235b_a22b_pretrain_config",
# Qwen3-Next models
"qwen3_next_80b_a3b_pretrain_config",
]
Loading