diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py index 3d5120b4d075..813e0a2f5f2e 100644 --- a/vllm/model_executor/models/commandr.py +++ b/vllm/model_executor/models/commandr.py @@ -348,7 +348,11 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): ".v_proj": (".qkv_proj", "v"), ".gate_proj": (".gate_up_proj", 0), ".up_proj": (".gate_up_proj", 1), - } + }, + # ModelOpt NVFP4 checkpoints carry raw quantizer-module state + # (e.g. "*.weight_quantizer._double_scale"); drop them before loading. + # See #41925. + orig_to_new_substr={"_quantizer.": None}, ) packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], @@ -356,9 +360,6 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): } # LoRA specific attributes embedding_modules = {"embed_tokens": "input_embeddings"} - # ModelOpt NVFP4 checkpoints carry raw quantizer-module state - # (e.g. "*.weight_quantizer._double_scale"); drop them before loading. See #41925. - hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={"_quantizer.": None}) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() diff --git a/vllm/model_executor/models/gemma3.py b/vllm/model_executor/models/gemma3.py index 717bc62439af..532a34d40fc0 100644 --- a/vllm/model_executor/models/gemma3.py +++ b/vllm/model_executor/models/gemma3.py @@ -291,6 +291,17 @@ def forward( @support_torch_compile class Gemma3Model(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config @@ -361,18 +372,13 @@ def forward( hidden_states, _ = self.norm(hidden_states, residual) return hidden_states + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_stacked={ - # weight_name: (param_name, shard_id) - ".q_proj": (".qkv_proj", "q"), - ".k_proj": (".qkv_proj", "k"), - ".v_proj": (".qkv_proj", "v"), - ".gate_proj": (".gate_up_proj", 0), - ".up_proj": (".gate_up_proj", 1), - } - ) + hf_to_vllm_mapper = Gemma3Model.hf_to_vllm_mapper packed_modules_mapping = { "qkv_proj": [ "q_proj", diff --git a/vllm/model_executor/models/jina.py b/vllm/model_executor/models/jina.py index 82a534404027..06f5ce282c6c 100644 --- a/vllm/model_executor/models/jina.py +++ b/vllm/model_executor/models/jina.py @@ -25,7 +25,7 @@ from .interfaces import SupportsLateInteraction from .interfaces_base import VllmModelForPooling from .qwen3 import Qwen3ForCausalLM, Qwen3Model -from .utils import AutoWeightsLoader, maybe_prefix +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix logger = logging.getLogger(__name__) @@ -193,6 +193,9 @@ class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling): """ is_pooling_model = True + hf_to_vllm_mapper = Qwen3ForCausalLM.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_prefix={"": "model."} + ) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__(vllm_config=vllm_config, prefix=prefix) @@ -254,6 +257,6 @@ def _merge_weights( tensor = tensor + (lora_B @ lora_A) * scaling yield name, tensor - loader = AutoWeightsLoader(self.model, ignore_unexpected_prefixes=["lm_head."]) + loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["lm_head."]) weights = _merge_weights(weights) - return loader.load_weights(weights, mapper=self.model.hf_to_vllm_mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)