[Model] Add tower and connector LoRA support for LFM2-VL

vllm/model_executor/models/lfm2_siglip2.py

@@ -20,6 +20,7 @@
 from vllm.model_executor.layers.linear import (
     ColumnParallelLinear,
     QKVParallelLinear,
+    ReplicatedLinear,
     RowParallelLinear,
 )
 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -37,9 +38,10 @@ def __init__(self, config: Siglip2VisionConfig):
         self.config = config
         self.embed_dim = config.hidden_size
         self.patch_size = config.patch_size
-        self.patch_embedding = nn.Linear(
-            in_features=config.num_channels * self.patch_size * self.patch_size,
-            out_features=self.embed_dim,
+        self.patch_embedding = ReplicatedLinear(
+            input_size=config.num_channels * self.patch_size * self.patch_size,
+            output_size=self.embed_dim,
+            return_bias=False,
         )
         self.num_patches = config.num_patches
         self.position_embedding_size = int(self.num_patches**0.5)

vllm/model_executor/models/lfm2_vl.py

@@ -22,6 +22,7 @@
 from vllm.config.multimodal import BaseDummyOptions
 from vllm.forward_context import set_forward_context
 from vllm.inputs import MultiModalDataDict
+from vllm.model_executor.layers.linear import ReplicatedLinear
 from vllm.model_executor.layers.mamba.mamba_utils import (
     MambaStateCopyFunc,
     MambaStateCopyFuncCalculator,
@@ -499,16 +500,20 @@ def __init__(
         self.projector_use_layernorm = config.projector_use_layernorm
         if self.projector_use_layernorm:
             self.layer_norm = nn.LayerNorm(in_channels)
-        self.linear_1 = nn.Linear(
+        self.linear_1 = ReplicatedLinear(
             in_channels,
             config.projector_hidden_size,
             bias=config.projector_bias,
+            prefix=maybe_prefix(prefix, "linear_1"),
+            return_bias=False,
         )
         self.act = ACT2FN[config.projector_hidden_act]
-        self.linear_2 = nn.Linear(
+        self.linear_2 = ReplicatedLinear(
             config.projector_hidden_size,
             config.text_config.hidden_size,
             bias=config.projector_bias,
+            prefix=maybe_prefix(prefix, "linear_2"),
+            return_bias=False,
         )
 
     def forward(
@@ -1259,3 +1264,13 @@ def get_mm_mapping(self) -> MultiModelKeys:
             connector="multi_modal_projector",
             tower_model="vision_tower",
         )
+
+    def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
+        downsample_factor = self.config.downsample_factor
+
+        return num_image_tokens * downsample_factor**2
+
+    def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
+        downsample_factor = self.config.downsample_factor
+
+        return num_vision_tokens // downsample_factor**2