diff --git a/vllm/lora/layers/base.py b/vllm/lora/layers/base.py index 85efbff1b03f..5fba2998c85a 100644 --- a/vllm/lora/layers/base.py +++ b/vllm/lora/layers/base.py @@ -15,6 +15,24 @@ class BaseLayerWithLoRA(nn.Module): + def __getattr__(self, name): + d = self.__dict__ + if name in d.get("_parameters", ()): + return d["_parameters"][name] + if name in d.get("_buffers", ()): + return d["_buffers"][name] + if name in d.get("_modules", ()): + return d["_modules"][name] + # Forward public misses to ``base_layer``; private names are framework + # bookkeeping and must stay local. + if not name.startswith("_"): + base_layer = d.get("_modules", {}).get("base_layer") + if base_layer is not None: + return getattr(base_layer, name) + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {name!r}" + ) + def load_weights( self, weights: Iterable[tuple[str, torch.Tensor]] ) -> Iterable[str]: diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index b5557f4b59e9..ad83203fc6f6 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -1001,6 +1001,8 @@ def make_expert_params_mapping( See `build_expert_params_mapping` for the returned tuple format. """ has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters()) + prefix = "base_layer." if has_base_layer else "" + # These loaders index ``params_dict[full_name]``, so both sides get it. return RoutedExperts.build_expert_params_mapping( ckpt_gate_proj_name, ckpt_down_proj_name, @@ -1008,7 +1010,8 @@ def make_expert_params_mapping( num_experts, num_redundant_experts, routed_experts_prefix, - "base_layer." if has_base_layer else "", + lora_base_layer_prefix=prefix, + lora_base_layer_prefix_on_param_name=prefix, ) @staticmethod @@ -1020,6 +1023,7 @@ def build_expert_params_mapping( num_redundant_experts: int = 0, routed_experts_prefix: str = "routed_experts", lora_base_layer_prefix: str = "", + lora_base_layer_prefix_on_param_name: str = "", include_fused: bool = False, ) -> list[tuple[str, str, int, str]]: """ @@ -1034,7 +1038,14 @@ def build_expert_params_mapping( ckpt_up_proj_name: Name of up projection in checkpoint num_experts: Number of logical (non-redundant) experts num_redundant_experts: Number of redundant experts - lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer + lora_base_layer_prefix: LoRA ``base_layer.`` prefix for the + ``weight_name`` (checkpoint) side + lora_base_layer_prefix_on_param_name: same, for the ``param_name`` + side. Independent because ``get_expert_mapping`` resolves + ``param_name`` via ``getattr`` against this layer's bare + ``w13_weight``/``w2_weight`` (no prefix), while + ``make_expert_params_mapping`` indexes the model-wide + ``params_dict`` (prefix included). include_fused: Prepend the fused pre-fused-checkpoint entries Returns: @@ -1060,8 +1071,10 @@ def build_expert_params_mapping( if routed_experts_prefix != "": routed_experts_prefix = f"{routed_experts_prefix}." - w13 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w13_" - w2 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w2_" + w13 = ( + f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w13_" + ) + w2 = f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w2_" fused_mapping = [] if include_fused: diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index 922d1c449871..171811becc6d 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -53,6 +53,7 @@ EagleModelMixin, MixtureOfExperts, SupportsEagle3, + SupportsLoRA, SupportsPP, ) from vllm.model_executor.models.utils import ( @@ -1236,6 +1237,11 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: if is_pp_missing_parameter(name, self): break + if name not in params_dict: + head, _, leaf = name.rpartition(".") + suffixed = f"{head}.base_layer.{leaf}" + if suffixed in params_dict: + name = suffixed param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) @@ -1290,6 +1296,13 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: else: if is_pp_missing_parameter(name, self): continue + # Non-LoRA params on a LoRA-wrapped module live at + # ``.base_layer.``; the checkpoint is plain. + if name not in params_dict: + head, _, leaf = name.rpartition(".") + suffixed = f"{head}.base_layer.{leaf}" + if suffixed in params_dict: + name = suffixed param = params_dict[name] weight_loader = getattr( param, "weight_loader", default_weight_loader @@ -1366,6 +1379,10 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper: # shared experts use Fp8LinearMethod's block scales, which # register as ``weight_scale_inv``. scale_regex = { + # ``.base_layer.``-namespace variant (LoRA-wrapped experts). + re.compile( + r"(\.experts\.\d+\.w[123]\.base_layer)\.scale$" + ): r"\1.weight_scale", re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale", re.compile(r"\.scale$"): ".weight_scale_inv", } @@ -1374,6 +1391,10 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper: # scales as ``w{13,2}_weight_scale_inv``. Map all ``.scale`` keys # there. scale_regex = { + # ``.base_layer.``-namespace variant of the above. + re.compile( + r"(\.experts\.\d+\.w[123]\.base_layer)\.scale$" + ): r"\1.weight_scale_inv", re.compile(r"\.scale$"): ".weight_scale_inv", } return WeightsMapper( @@ -1434,7 +1455,11 @@ def update_physical_experts_metadata( class DeepseekV4ForCausalLM( - nn.Module, SupportsPP, SupportsEagle3, DeepseekV4MixtureOfExperts + nn.Module, + SupportsPP, + SupportsEagle3, + SupportsLoRA, + DeepseekV4MixtureOfExperts, ): model_cls = DeepseekV4Model @@ -1442,6 +1467,15 @@ class DeepseekV4ForCausalLM( # Overridden per-instance in __init__ when expert_dtype != "fp4". hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4") + packed_modules_mapping = { + "gate_up_proj": ["w1", "w3"], + "fused_wqa_wkv": ["wq_a", "wkv"], + "fused_wkv_wgate": ["wkv", "wgate"], + } + + # The MTP draft head is not LoRA-adapted. + lora_skip_prefixes = ["mtp."] + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__()