From 30a8e4c6cf73730151648131f2e37c036caf2c98 Mon Sep 17 00:00:00 2001 From: Hollow Man Date: Wed, 19 Aug 2026 20:39:23 +0300 Subject: [PATCH 1/2] [LoRA] feat: Support LoRA for DeepSeek V4 Add LoRA support for DeepSeek V4 (DSV4). DSV4 is a MoE model with quantized (fp8/mxfp4) experts, MLA, and an MTP draft head. Three issues blocked merge=False (live-adapter) LoRA, causing train/inference inconsistency: 1. **Non-LoRA params on a LoRA-wrapped module** (e.g. `gate.tid2eid`, `gate.e_score_correction_bias`) live at `.base_layer.` in the live namespace, but the checkpoint name is plain. The DSV4 `load_weights` strict `params_dict[name]` lookup KeyError'd / orphaned these on the initial profile load. 2. **Expert mapping `param_name` vs `weight_name` prefix confusion.** `RoutedExperts.make_expert_params_mapping` applied a single `lora_base_layer_prefix` to both sides. But `get_expert_mapping` resolves `param_name` via `getattr` against this layer's bare `w13_weight`/`w2_weight` (no prefix), while `make_expert_params_mapping` indexes the model-wide `params_dict` (prefix included). One prefix for both sides silently dropped per-expert weights. 3. **`weight_scale_inv` / scale not forwarded through the LoRA wrapper.** Custom kernels read `weight_scale_inv` through the wrapper; without `__getattr__` forwarding the attribute lookup failed. This PR adds `SupportsLoRA` to `DeepseekV4ForCausalLM`, declares `packed_modules_mapping` and `lora_skip_prefixes` (the MTP draft head is not LoRA-adapted), adds `.base_layer.`-namespace variants to the weights mapper scale regexes, reconciles non-LoRA params in the inner `load_weights`, splits the expert mapping prefix into checkpoint-side (`weight_name`) and live-side (`param_name`), and forwards public attribute misses in `BaseLayerWithLoRA` to `base_layer`. Signed-off-by: Hollow Man --- vllm/lora/layers/base.py | 18 +++++++++++ .../layers/fused_moe/routed_experts.py | 21 ++++++++++--- vllm/models/deepseek_v4/nvidia/model.py | 31 ++++++++++++++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/vllm/lora/layers/base.py b/vllm/lora/layers/base.py index 85efbff1b03f..5fba2998c85a 100644 --- a/vllm/lora/layers/base.py +++ b/vllm/lora/layers/base.py @@ -15,6 +15,24 @@ class BaseLayerWithLoRA(nn.Module): + def __getattr__(self, name): + d = self.__dict__ + if name in d.get("_parameters", ()): + return d["_parameters"][name] + if name in d.get("_buffers", ()): + return d["_buffers"][name] + if name in d.get("_modules", ()): + return d["_modules"][name] + # Forward public misses to ``base_layer``; private names are framework + # bookkeeping and must stay local. + if not name.startswith("_"): + base_layer = d.get("_modules", {}).get("base_layer") + if base_layer is not None: + return getattr(base_layer, name) + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {name!r}" + ) + def load_weights( self, weights: Iterable[tuple[str, torch.Tensor]] ) -> Iterable[str]: diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index b5557f4b59e9..ad83203fc6f6 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -1001,6 +1001,8 @@ def make_expert_params_mapping( See `build_expert_params_mapping` for the returned tuple format. """ has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters()) + prefix = "base_layer." if has_base_layer else "" + # These loaders index ``params_dict[full_name]``, so both sides get it. return RoutedExperts.build_expert_params_mapping( ckpt_gate_proj_name, ckpt_down_proj_name, @@ -1008,7 +1010,8 @@ def make_expert_params_mapping( num_experts, num_redundant_experts, routed_experts_prefix, - "base_layer." if has_base_layer else "", + lora_base_layer_prefix=prefix, + lora_base_layer_prefix_on_param_name=prefix, ) @staticmethod @@ -1020,6 +1023,7 @@ def build_expert_params_mapping( num_redundant_experts: int = 0, routed_experts_prefix: str = "routed_experts", lora_base_layer_prefix: str = "", + lora_base_layer_prefix_on_param_name: str = "", include_fused: bool = False, ) -> list[tuple[str, str, int, str]]: """ @@ -1034,7 +1038,14 @@ def build_expert_params_mapping( ckpt_up_proj_name: Name of up projection in checkpoint num_experts: Number of logical (non-redundant) experts num_redundant_experts: Number of redundant experts - lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer + lora_base_layer_prefix: LoRA ``base_layer.`` prefix for the + ``weight_name`` (checkpoint) side + lora_base_layer_prefix_on_param_name: same, for the ``param_name`` + side. Independent because ``get_expert_mapping`` resolves + ``param_name`` via ``getattr`` against this layer's bare + ``w13_weight``/``w2_weight`` (no prefix), while + ``make_expert_params_mapping`` indexes the model-wide + ``params_dict`` (prefix included). include_fused: Prepend the fused pre-fused-checkpoint entries Returns: @@ -1060,8 +1071,10 @@ def build_expert_params_mapping( if routed_experts_prefix != "": routed_experts_prefix = f"{routed_experts_prefix}." - w13 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w13_" - w2 = f"experts.{lora_base_layer_prefix}{routed_experts_prefix}w2_" + w13 = ( + f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w13_" + ) + w2 = f"experts.{lora_base_layer_prefix_on_param_name}{routed_experts_prefix}w2_" fused_mapping = [] if include_fused: diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index 922d1c449871..f2ddba26f004 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -53,6 +53,7 @@ EagleModelMixin, MixtureOfExperts, SupportsEagle3, + SupportsLoRA, SupportsPP, ) from vllm.model_executor.models.utils import ( @@ -1290,6 +1291,13 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: else: if is_pp_missing_parameter(name, self): continue + # Non-LoRA params on a LoRA-wrapped module live at + # ``.base_layer.``; the checkpoint is plain. + if name not in params_dict: + head, _, leaf = name.rpartition(".") + suffixed = f"{head}.base_layer.{leaf}" + if suffixed in params_dict: + name = suffixed param = params_dict[name] weight_loader = getattr( param, "weight_loader", default_weight_loader @@ -1366,6 +1374,10 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper: # shared experts use Fp8LinearMethod's block scales, which # register as ``weight_scale_inv``. scale_regex = { + # ``.base_layer.``-namespace variant (LoRA-wrapped experts). + re.compile( + r"(\.experts\.\d+\.w[123]\.base_layer)\.scale$" + ): r"\1.weight_scale", re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale", re.compile(r"\.scale$"): ".weight_scale_inv", } @@ -1374,6 +1386,10 @@ def _make_deepseek_v4_weights_mapper(expert_dtype: str) -> WeightsMapper: # scales as ``w{13,2}_weight_scale_inv``. Map all ``.scale`` keys # there. scale_regex = { + # ``.base_layer.``-namespace variant of the above. + re.compile( + r"(\.experts\.\d+\.w[123]\.base_layer)\.scale$" + ): r"\1.weight_scale_inv", re.compile(r"\.scale$"): ".weight_scale_inv", } return WeightsMapper( @@ -1434,7 +1450,11 @@ def update_physical_experts_metadata( class DeepseekV4ForCausalLM( - nn.Module, SupportsPP, SupportsEagle3, DeepseekV4MixtureOfExperts + nn.Module, + SupportsPP, + SupportsEagle3, + SupportsLoRA, + DeepseekV4MixtureOfExperts, ): model_cls = DeepseekV4Model @@ -1442,6 +1462,15 @@ class DeepseekV4ForCausalLM( # Overridden per-instance in __init__ when expert_dtype != "fp4". hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4") + packed_modules_mapping = { + "gate_up_proj": ["w1", "w3"], + "fused_wqa_wkv": ["wq_a", "wkv"], + "fused_wkv_wgate": ["wkv", "wgate"], + } + + # The MTP draft head is not LoRA-adapted. + lora_skip_prefixes = ["mtp."] + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() From 56c9afb36892c5dce10873460c01873784eb1352 Mon Sep 17 00:00:00 2001 From: Hollow Man Date: Fri, 21 Aug 2026 08:38:59 +0300 Subject: [PATCH 2/2] additional base_layer check Signed-off-by: Hollow Man --- vllm/models/deepseek_v4/nvidia/model.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index f2ddba26f004..171811becc6d 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -1237,6 +1237,11 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: if is_pp_missing_parameter(name, self): break + if name not in params_dict: + head, _, leaf = name.rpartition(".") + suffixed = f"{head}.base_layer.{leaf}" + if suffixed in params_dict: + name = suffixed param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id)