Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions src/megatron/bridge/models/conversion/peft_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from __future__ import annotations

import itertools
import re
from collections import defaultdict
from dataclasses import dataclass
from string import digits
Expand Down Expand Up @@ -865,6 +866,83 @@ def stream_adapter_weights_megatron_to_hf(
megatron_linear_out_name = adapter_weight.linear_out_weight.param_name
is_expert = is_expert_linear(adapter_task.global_base_prefix)
is_grouped_expert = is_expert and ".local_experts." not in adapter_task.global_base_prefix
# A grouped-expert adapter whose two sides have different ndim is a
# shared-outer LoRA (SGLang PR #21466): one side is the shared 2D
# LoRA matrix, the other is the per-expert 3D pack.
is_shared_outer_lora = is_grouped_expert and linear_in_tensor.ndim != linear_out_tensor.ndim

# The shared side is replicated on every EP rank, so the default
# gather is skipped here and re-done per-side inside the branch.
if is_shared_outer_lora:
for side_tensor, side_suffix, megatron_side_name in (
(linear_in_tensor, ".linear_in.weight", megatron_linear_in_name),
(linear_out_tensor, ".linear_out.weight", megatron_linear_out_name),
):
side_is_shared = side_tensor.ndim == 2
side_gathered = (
None if side_is_shared else self._gather_expert_adapter_weight(side_tensor)
)
side_suffixes = (
[".weight0"] if side_is_shared
else [f".weight{i}" for i in range(num_moe_experts)]
)
for base_suffix in side_suffixes:
if side_is_shared:
current = side_tensor
else:
expert_idx = int(base_suffix[len(".weight"):])
current = self._select_expert_adapter_weight(
side_tensor, side_gathered, expert_idx, num_moe_experts,
)
if cpu:
current = current.cpu()
if side_is_shared:
current = current.unsqueeze(0) # 2D -> [1, ..., ...]

base_hf_weight_names = self._get_base_hf_param_names_for_adapter(
mapping_registry,
adapter_task.global_base_prefix,
adapter_task.adapter_key,
base_suffix,
)
if side_is_shared:
# The shared tensor is not "expert N"'s weight;
# strip the experts.N. infix so the serving loader
# takes the 3D-shared branch rather than the
# per-expert dict path.
base_hf_weight_names = [
re.sub(r"\bexperts\.\d+\.", "experts.", n)
for n in base_hf_weight_names
]
side_hf_names = [
self._make_lora_param_name(bn, side_suffix) for bn in base_hf_weight_names
]

if side_suffix == ".linear_out.weight" and adapter_task.adapter_key is None:
per_base = self._get_fused_adapter_linear_out_slices(
megatron_model,
base_hf_weight_names,
current,
is_expert=is_expert_linear(adapter_task.global_base_prefix),
)
if per_base is not None:
for index, base_name in enumerate(base_hf_weight_names):
chunk = per_base.get(base_name)
assert chunk is not None, f"unknown projection name: {base_name!r}"
yield HFWeightTuple(side_hf_names[index], chunk, megatron_side_name)
continue

# Shared linear_in of a fused linear_fc1 feeds BOTH gate
# and up projections; the same tensor must appear at both
# HF names so gate/up stacking sees a matching pair.
if side_is_shared and len(side_hf_names) > 1:
for hf_name in side_hf_names:
yield HFWeightTuple(hf_name, current, megatron_side_name)
continue

yield HFWeightTuple(side_hf_names[0], current, megatron_side_name)
continue

expert_linear_in_gathered = None
expert_linear_out_gathered = None
if is_grouped_expert:
Expand Down
4 changes: 3 additions & 1 deletion src/megatron/bridge/models/kimi_vl/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,9 @@ def quantize_to_int4(
weight_shape: Original tensor shape (int32)
"""
out_features, in_features = weight.shape
weight_shape = torch.tensor([out_features, in_features], dtype=torch.int32)
weight_shape = torch.tensor(
[out_features, in_features], dtype=torch.int32, device=weight.device
)

# Convert to float32 for computation
w = weight.float()
Expand Down
28 changes: 24 additions & 4 deletions src/megatron/bridge/peft/lora.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
from megatron.bridge.peft.utils import (
GroupedExpertLinearAdapter,
ParallelLinearAdapter,
SharedOuterGroupedExpertAdapter,
align_expert_dim_for_tp,
get_adapter_attributes_from_linear,
get_effective_lora_dim,
Expand Down Expand Up @@ -92,6 +93,13 @@ class LoRA(PEFT, ModuleMatcher):
so it is comparable to a dense model. Defaults to False.
share_expert_adapters (bool): When True, grouped MoE expert linears share one adapter across all local
experts on the EP rank. Set to False to create one adapter per local expert instead. Defaults to True.
experts_shared_outer_loras (bool): When True, grouped-expert LoRA
(``TE*ParallelGroupedLinear`` base modules) uses
:class:`SharedOuterGroupedExpertAdapter` — ``gate_up`` lora_A and
``down`` lora_B are shared across experts (expert_dim=1), matching
SGLang's ``experts_shared_outer_loras=True`` serving contract (PR
#21466). Default False preserves the adapter layout selected by
``share_expert_adapters``.
"""

target_modules: List[str] = field(
Expand All @@ -107,6 +115,7 @@ class LoRA(PEFT, ModuleMatcher):
lora_dtype: torch.dtype = None
normalize_moe_lora: bool = False
share_expert_adapters: bool = True
experts_shared_outer_loras: bool = False

def transform(self, module: nn.Module, name: Optional[str] = None, prefix: Optional[str] = None) -> nn.Module:
"""
Expand Down Expand Up @@ -165,18 +174,29 @@ def transform(self, module: nn.Module, name: Optional[str] = None, prefix: Optio
is_expert=is_expert,
input_is_parallel=attrs.input_is_parallel,
)
use_per_expert_adapter = is_grouped_expert_linear(full_name) and not self.share_expert_adapters
is_grouped_expert_name = is_grouped_expert_linear(full_name)
use_shared_outer_adapter = self.experts_shared_outer_loras and is_grouped_expert_name
use_per_expert_adapter = (
is_grouped_expert_name and not self.share_expert_adapters and not use_shared_outer_adapter
)
use_grouped_expert_adapter = use_shared_outer_adapter or use_per_expert_adapter

enable_op_fuser = (
not use_per_expert_adapter
not use_grouped_expert_adapter
and not is_expert
and getattr(module.config, "use_transformer_engine_op_fuser", False)
# TP not yet supported
and parallel_state.get_tensor_model_parallel_world_size() == 1
)

logger.info(f"Adding lora to: {full_name}")
adapter_cls = GroupedExpertLinearAdapter if use_per_expert_adapter else ParallelLinearAdapter

if use_shared_outer_adapter:
adapter_cls = SharedOuterGroupedExpertAdapter
elif use_per_expert_adapter:
adapter_cls = GroupedExpertLinearAdapter
else:
adapter_cls = ParallelLinearAdapter
adapter_kwargs = dict(
base_linear_name=full_name,
activation="identity",
Expand All @@ -189,7 +209,7 @@ def transform(self, module: nn.Module, name: Optional[str] = None, prefix: Optio
alpha=self.alpha,
base_linear_is_parallel=attrs.base_linear_is_parallel,
)
if use_per_expert_adapter:
if use_grouped_expert_adapter:
first_param = next(module.parameters())
adapter_kwargs.update(
num_local_experts=module.num_gemms,
Expand Down
Loading
Loading