From 88246da0306d84e36a70e63ded24ebe86289b3b2 Mon Sep 17 00:00:00 2001 From: zupengwang <71580390+zupengwang@users.noreply.github.com> Date: Sat, 8 Aug 2026 19:00:22 +0800 Subject: [PATCH 1/2] [Model] Add tower and connector LoRA support for LFM2-VL Convert the LFM2-VL projector linears to LoRA-wrappable replicated layers and add token budget helpers based on the projector downsample factor. Add focused coverage for the real 450M config, LoRA layer replacement, round trips, and zero-token inputs. Assisted-by: OpenAI Codex Signed-off-by: zupengwang <71580390+zupengwang@users.noreply.github.com> --- .../multimodal/processing/test_lfm2_vl.py | 88 +++++++++++++++++++ vllm/model_executor/models/lfm2_vl.py | 19 +++- 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 tests/models/multimodal/processing/test_lfm2_vl.py diff --git a/tests/models/multimodal/processing/test_lfm2_vl.py b/tests/models/multimodal/processing/test_lfm2_vl.py new file mode 100644 index 000000000000..9281912506f8 --- /dev/null +++ b/tests/models/multimodal/processing/test_lfm2_vl.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest + +from vllm.config.lora import LoRAConfig +from vllm.lora.layers import ReplicatedLinearWithLoRA +from vllm.model_executor.models.lfm2_vl import ( + Lfm2VLForConditionalGeneration, + Lfm2VLMultiModalProjector, +) + +from ...utils import build_model_context + +get_num_mm_encoder_tokens = Lfm2VLForConditionalGeneration.get_num_mm_encoder_tokens +get_num_mm_connector_tokens = Lfm2VLForConditionalGeneration.get_num_mm_connector_tokens + + +@pytest.mark.parametrize("model_id", ["LiquidAI/LFM2-VL-450M"]) +def test_num_mm_tokens_match_real_config( + model_id, + monkeypatch: pytest.MonkeyPatch, +): + ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 1}) + config = ctx.model_config.hf_config + stub = SimpleNamespace(config=config) + factor = config.downsample_factor + monkeypatch.setattr( + "vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", + lambda: 0, + ) + monkeypatch.setattr( + "vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", + lambda: 1, + ) + monkeypatch.setattr( + "vllm.model_executor.parameter.get_tensor_model_parallel_rank", + lambda: 0, + ) + monkeypatch.setattr( + "vllm.model_executor.parameter.get_tensor_model_parallel_world_size", + lambda: 1, + ) + projector = Lfm2VLMultiModalProjector(config) + lora_config = LoRAConfig(max_loras=1, max_lora_rank=8) + + for layer in (projector.linear_1, projector.linear_2): + assert ReplicatedLinearWithLoRA.can_replace_layer( + source_layer=layer, + lora_config=lora_config, + packed_modules_list=[], + model_config=config, + ) + + for num_image_tokens in (1, 17, 256, 1024): + encoder_tokens = get_num_mm_encoder_tokens(stub, num_image_tokens) + assert encoder_tokens == num_image_tokens * factor**2 + + connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) + assert connector_tokens == num_image_tokens + + +@pytest.mark.parametrize( + ("downsample_factor", "num_image_tokens"), + [ + (1, 1), + (2, 17), + (3, 256), + (4, 1024), + ], +) +def test_num_mm_tokens_roundtrip(downsample_factor, num_image_tokens): + stub = SimpleNamespace(config=SimpleNamespace(downsample_factor=downsample_factor)) + + encoder_tokens = get_num_mm_encoder_tokens(stub, num_image_tokens) + assert encoder_tokens == num_image_tokens * downsample_factor**2 + + connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) + assert connector_tokens == num_image_tokens + + +def test_num_mm_tokens_zero(): + stub = SimpleNamespace(config=SimpleNamespace(downsample_factor=2)) + + assert get_num_mm_encoder_tokens(stub, 0) == 0 + assert get_num_mm_connector_tokens(stub, 0) == 0 diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index ce60f2d236d9..d247fef4d7e3 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -22,6 +22,7 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.forward_context import set_forward_context from vllm.inputs import MultiModalDataDict +from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFunc, MambaStateCopyFuncCalculator, @@ -499,16 +500,20 @@ def __init__( self.projector_use_layernorm = config.projector_use_layernorm if self.projector_use_layernorm: self.layer_norm = nn.LayerNorm(in_channels) - self.linear_1 = nn.Linear( + self.linear_1 = ReplicatedLinear( in_channels, config.projector_hidden_size, bias=config.projector_bias, + prefix=maybe_prefix(prefix, "linear_1"), + return_bias=False, ) self.act = ACT2FN[config.projector_hidden_act] - self.linear_2 = nn.Linear( + self.linear_2 = ReplicatedLinear( config.projector_hidden_size, config.text_config.hidden_size, bias=config.projector_bias, + prefix=maybe_prefix(prefix, "linear_2"), + return_bias=False, ) def forward( @@ -1259,3 +1264,13 @@ def get_mm_mapping(self) -> MultiModelKeys: connector="multi_modal_projector", tower_model="vision_tower", ) + + def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + downsample_factor = self.config.downsample_factor + + return num_image_tokens * downsample_factor**2 + + def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: + downsample_factor = self.config.downsample_factor + + return num_vision_tokens // downsample_factor**2 From 195e139807c3af4490e083ddc54200b501a9e9b3 Mon Sep 17 00:00:00 2001 From: zupengwang <71580390+zupengwang@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:49:08 +0800 Subject: [PATCH 2/2] [Model] Address LFM2-VL LoRA review feedback Remove the standalone LFM2-VL processing test file per review. Convert the Siglip2 patch projection to ReplicatedLinear so tower LoRA targets are discoverable and wrappable while preserving its tensor-only forward contract. Assisted-by: OpenAI Codex Signed-off-by: zupengwang <71580390+zupengwang@users.noreply.github.com> --- .../multimodal/processing/test_lfm2_vl.py | 88 ------------------- vllm/model_executor/models/lfm2_siglip2.py | 8 +- 2 files changed, 5 insertions(+), 91 deletions(-) delete mode 100644 tests/models/multimodal/processing/test_lfm2_vl.py diff --git a/tests/models/multimodal/processing/test_lfm2_vl.py b/tests/models/multimodal/processing/test_lfm2_vl.py deleted file mode 100644 index 9281912506f8..000000000000 --- a/tests/models/multimodal/processing/test_lfm2_vl.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from types import SimpleNamespace - -import pytest - -from vllm.config.lora import LoRAConfig -from vllm.lora.layers import ReplicatedLinearWithLoRA -from vllm.model_executor.models.lfm2_vl import ( - Lfm2VLForConditionalGeneration, - Lfm2VLMultiModalProjector, -) - -from ...utils import build_model_context - -get_num_mm_encoder_tokens = Lfm2VLForConditionalGeneration.get_num_mm_encoder_tokens -get_num_mm_connector_tokens = Lfm2VLForConditionalGeneration.get_num_mm_connector_tokens - - -@pytest.mark.parametrize("model_id", ["LiquidAI/LFM2-VL-450M"]) -def test_num_mm_tokens_match_real_config( - model_id, - monkeypatch: pytest.MonkeyPatch, -): - ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 1}) - config = ctx.model_config.hf_config - stub = SimpleNamespace(config=config) - factor = config.downsample_factor - monkeypatch.setattr( - "vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", - lambda: 0, - ) - monkeypatch.setattr( - "vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", - lambda: 1, - ) - monkeypatch.setattr( - "vllm.model_executor.parameter.get_tensor_model_parallel_rank", - lambda: 0, - ) - monkeypatch.setattr( - "vllm.model_executor.parameter.get_tensor_model_parallel_world_size", - lambda: 1, - ) - projector = Lfm2VLMultiModalProjector(config) - lora_config = LoRAConfig(max_loras=1, max_lora_rank=8) - - for layer in (projector.linear_1, projector.linear_2): - assert ReplicatedLinearWithLoRA.can_replace_layer( - source_layer=layer, - lora_config=lora_config, - packed_modules_list=[], - model_config=config, - ) - - for num_image_tokens in (1, 17, 256, 1024): - encoder_tokens = get_num_mm_encoder_tokens(stub, num_image_tokens) - assert encoder_tokens == num_image_tokens * factor**2 - - connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) - assert connector_tokens == num_image_tokens - - -@pytest.mark.parametrize( - ("downsample_factor", "num_image_tokens"), - [ - (1, 1), - (2, 17), - (3, 256), - (4, 1024), - ], -) -def test_num_mm_tokens_roundtrip(downsample_factor, num_image_tokens): - stub = SimpleNamespace(config=SimpleNamespace(downsample_factor=downsample_factor)) - - encoder_tokens = get_num_mm_encoder_tokens(stub, num_image_tokens) - assert encoder_tokens == num_image_tokens * downsample_factor**2 - - connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) - assert connector_tokens == num_image_tokens - - -def test_num_mm_tokens_zero(): - stub = SimpleNamespace(config=SimpleNamespace(downsample_factor=2)) - - assert get_num_mm_encoder_tokens(stub, 0) == 0 - assert get_num_mm_connector_tokens(stub, 0) == 0 diff --git a/vllm/model_executor/models/lfm2_siglip2.py b/vllm/model_executor/models/lfm2_siglip2.py index f1679af813c4..31e7c0f14438 100644 --- a/vllm/model_executor/models/lfm2_siglip2.py +++ b/vllm/model_executor/models/lfm2_siglip2.py @@ -20,6 +20,7 @@ from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, + ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig @@ -37,9 +38,10 @@ def __init__(self, config: Siglip2VisionConfig): self.config = config self.embed_dim = config.hidden_size self.patch_size = config.patch_size - self.patch_embedding = nn.Linear( - in_features=config.num_channels * self.patch_size * self.patch_size, - out_features=self.embed_dim, + self.patch_embedding = ReplicatedLinear( + input_size=config.num_channels * self.patch_size * self.patch_size, + output_size=self.embed_dim, + return_bias=False, ) self.num_patches = config.num_patches self.position_embedding_size = int(self.num_patches**0.5)