From 66acef20fc59a3d50ef7ca811f463b571aff0dee Mon Sep 17 00:00:00 2001 From: Reagan Date: Mon, 29 Dec 2025 13:07:53 -0800 Subject: [PATCH 1/4] change Signed-off-by: Reagan --- tests/models/utils.py | 10 +++++- tests/multimodal/test_registry.py | 52 +++++++++++++++++++++++---- vllm/multimodal/processing.py | 6 ++++ vllm/multimodal/registry.py | 11 +++--- vllm/v1/core/encoder_cache_manager.py | 17 +++++++++ vllm/v1/worker/gpu_model_runner.py | 37 +++++++++++++++++-- vllm/v1/worker/tpu_model_runner.py | 36 +++++++++++++++++-- vllm/v1/worker/utils.py | 1 + 8 files changed, 154 insertions(+), 16 deletions(-) diff --git a/tests/models/utils.py b/tests/models/utils.py index d84b4b820533..976332721c72 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -278,6 +278,7 @@ def build_model_context( mm_processor_kwargs: dict[str, Any] | None = None, limit_mm_per_prompt: dict[str, int] | None = None, mm_processor_cache_gb: int = 0, + enable_mm_embeds: bool | None = None, ): """Creates an InputProcessingContext for a given model. @@ -286,6 +287,8 @@ def build_model_context( mm_processor_kwargs: optional processor kwargs for to be leveraged in the input processor, mapper, dummy data creation, etc. limit_mm_per_prompt: Multimodal limits. + enable_mm_embeds: Whether to enable multimodal embedding processing. + If None, defaults to model_info.require_embed_inputs. Returns: InputProcessingContext for the model being considered. @@ -296,6 +299,11 @@ def build_model_context( model_config_kwargs = model_config_kwargs or {} limit_mm_per_prompt = limit_mm_per_prompt or {} + enable_mm_embeds = ( + model_info.require_embed_inputs + if enable_mm_embeds is None + else enable_mm_embeds + ) model_config = ModelConfig( model_id, runner=runner, @@ -311,7 +319,7 @@ def build_model_context( hf_overrides=model_info.hf_overrides, skip_tokenizer_init=model_info.require_embed_inputs, enable_prompt_embeds=model_info.require_embed_inputs, - enable_mm_embeds=model_info.require_embed_inputs, + enable_mm_embeds=enable_mm_embeds, enforce_eager=model_info.enforce_eager, **model_config_kwargs, ) diff --git a/tests/multimodal/test_registry.py b/tests/multimodal/test_registry.py index 3b01bda7f54c..cf1705464197 100644 --- a/tests/multimodal/test_registry.py +++ b/tests/multimodal/test_registry.py @@ -15,20 +15,58 @@ @pytest.mark.parametrize( - "model_id,limit_mm_per_prompt,expected", + "model_id,limit_mm_per_prompt,enable_mm_embeds,expected", [ - ("Qwen/Qwen2-0.5B-Instruct", {}, False), - ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True), - ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False), - ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True), + ("Qwen/Qwen2-0.5B-Instruct", {}, False, False), + ("Qwen/Qwen2-0.5B-Instruct", {}, True, False), + ("Qwen/Qwen2.5-VL-3B-Instruct", {}, False, True), + ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True, True), + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False, False), + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, True, True), + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True, True), + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, False, True), + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 1, "video": 1}, True, True), + ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 1, "video": 1}, False, True), ], ) @pytest.mark.core_model -def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected): +def test_supports_multimodal_inputs( + model_id, limit_mm_per_prompt, enable_mm_embeds, expected +): """Test supports_multimodal_inputs returns correct boolean for various - configs.""" + configs. + + This test verifies that when enable_mm_embeds=True, the multimodal + processing pipeline is enabled even when all modality limits are 0, + allowing pre-computed embeddings to be processed while skipping + encoder weight loading for memory optimization. + """ ctx = build_model_context( model_id, limit_mm_per_prompt=limit_mm_per_prompt, + enable_mm_embeds=enable_mm_embeds, ) assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.model_config) is expected + + +@pytest.mark.core_model +def test_encoder_weight_skipping_with_enable_mm_embeds(): + """Integration test verifying encoder weights are not loaded when all + modality limits are 0, even with enable_mm_embeds=True.""" + from vllm.v1.core.encoder_cache_manager import compute_encoder_cache_budget + + ctx = build_model_context( + "Qwen/Qwen2.5-VL-3B-Instruct", + limit_mm_per_prompt={"image": 0, "video": 0}, + enable_mm_embeds=True, + ) + + assert MULTIMODAL_REGISTRY.supports_multimodal_inputs(ctx.model_config) is True + + compute_budget, cache_size = compute_encoder_cache_budget( + ctx.model_config, ctx.scheduler_config, MULTIMODAL_REGISTRY + ) + assert compute_budget == 1, ( + "Compute budget should be minimal (1) for embeddings-only mode" + ) + assert cache_size == 1, "Cache size should be minimal (1) for embeddings-only mode" \ No newline at end of file diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing.py index 3bbdab3b393c..057b4951f9ad 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing.py @@ -1384,6 +1384,12 @@ def _to_mm_items( ) for modality, items in mm_items.items(): + # When enable_mm_embeds=True, allow embedding inputs even if + # modality limits are zero (for memory optimization) + if mm_config.enable_mm_embeds and isinstance( + items, (EmbeddingItems, DictEmbeddingItems) + ): + continue self.validate_num_items(modality, len(items)) return mm_items diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index 1e7fe8648ab7..f179416457fe 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -118,17 +118,20 @@ def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool: """ Checks if the model supports multimodal inputs. Returns True if the model is multimodal with any non-zero supported - modalities, otherwise returns False, effectively running in - text-only mode. + modalities OR if enable_mm_embeds is True (for pre-computed embeddings), + otherwise returns False, effectively running in text-only mode. """ if not model_config.is_multimodal_model: return False + mm_config = model_config.get_multimodal_config() + + if mm_config.enable_mm_embeds: + return True + info = self._create_processing_info(model_config, tokenizer=None) supported_modalities = info.get_supported_mm_limits() - mm_config = model_config.get_multimodal_config() - # Check if all supported modalities have limit == 0 if all( mm_config.get_limit_per_prompt(modality) == 0 diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index d73c05d2cf80..c5f47ba2d9ca 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -277,6 +277,7 @@ def compute_encoder_budget( return compute_mm_encoder_budget( scheduler_config, max_tokens_by_modality, + model_config, ) return compute_text_encoder_budget(scheduler_config) @@ -302,6 +303,7 @@ def compute_text_encoder_budget(scheduler_config: "SchedulerConfig") -> tuple[in def compute_mm_encoder_budget( scheduler_config: "SchedulerConfig", max_tokens_by_modality: Mapping[str, int], + model_config: "ModelConfig | None" = None, ) -> tuple[int, int]: """Compute the encoder cache budget based on the model and scheduler configurations for a multimodal model. @@ -310,6 +312,7 @@ def compute_mm_encoder_budget( scheduler_config: Scheduler configuration. max_tokens_by_modality: The maximum number of tokens for each non-text modality. + model_config: Model configuration. Used to check enable_mm_embeds flag. Returns: - Compute budget for encoder execution, measured in number of tokens @@ -319,6 +322,20 @@ def compute_mm_encoder_budget( """ if not max_tokens_by_modality: + # Check if enable_mm_embeds is True - if so, return minimal cache size + # to allow None value caching for memory optimization + if ( + model_config is not None + and model_config.multimodal_config is not None + and model_config.multimodal_config.enable_mm_embeds + ): + logger.info( + "All non-text modalities are disabled but enable_mm_embeds=True. " + "Initializing encoder cache with minimal size to support " + "embedding inputs." + ) + return 1, 1 + logger.warning( "All non-text modalities supported by the model have been " "explicitly disabled via limit_mm_per_prompt. Encoder cache will " diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 92822d829a88..7a55adc3be5a 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2267,7 +2267,25 @@ def _gather_mm_embeddings( mm_hash = mm_feature.identifier encoder_output = self.encoder_cache.get(mm_hash, None) - assert encoder_output is not None, f"Encoder cache miss for {mm_hash}." + + if encoder_output is None: + mm_config = self.model_config.multimodal_config + if ( + mm_config is not None + and mm_config.enable_mm_embeds + and "dummy_none" in self.encoder_cache + and mm_hash not in self.encoder_cache + ): + # enable_mm_embeds is True and we have the dummy cache entry. + # This edge case can occur when embeddings are provided + # directly without going through the normal encoder caching + # path. Skip this entry as the embeddings may have been + # provided directly. + continue + else: + assert encoder_output is not None, ( + f"Encoder cache miss for {mm_hash}." + ) if (is_embed := pos_info.is_embed) is not None: is_embed = is_embed[start_idx:end_idx] @@ -4506,7 +4524,22 @@ def profile_run(self) -> None: mm_budget = self.mm_budget assert mm_budget is not None - if (encoder_budget := mm_budget.get_encoder_budget()) > 0: + encoder_budget = mm_budget.get_encoder_budget() + # Check if this is a minimal budget case (encoder_budget == 1) + # which occurs when all modalities are zero but enable_mm_embeds=True + if ( + encoder_budget == 1 + and mm_config is not None + and mm_config.enable_mm_embeds + and not mm_budget.max_tokens_by_modality + ): + logger.info( + "All modalities are disabled but enable_mm_embeds=True. " + "Initializing encoder cache with None values for memory " + "optimization." + ) + self.encoder_cache["dummy_none"] = None + elif encoder_budget > 0: # NOTE: Currently model is profiled with a single non-text # modality with the max possible input tokens even when # it supports multiple. diff --git a/vllm/v1/worker/tpu_model_runner.py b/vllm/v1/worker/tpu_model_runner.py index c7404c4642d7..68cae47b275a 100644 --- a/vllm/v1/worker/tpu_model_runner.py +++ b/vllm/v1/worker/tpu_model_runner.py @@ -1052,7 +1052,24 @@ def _gather_mm_embeddings( mm_hash = mm_feature.identifier encoder_output = self.encoder_cache.get(mm_hash, None) - assert encoder_output is not None, f"Encoder cache miss for {mm_hash}." + if encoder_output is None: + mm_config = self.model_config.multimodal_config + if ( + mm_config is not None + and mm_config.enable_mm_embeds + and "dummy_none" in self.encoder_cache + and mm_hash not in self.encoder_cache + ): + # enable_mm_embeds is True and we have the dummy cache entry. + # This edge case can occur when embeddings are provided + # directly without going through the normal encoder caching + # path. Skip this entry as the embeddings may have been + # provided directly. + continue + else: + assert encoder_output is not None, ( + f"Encoder cache miss for {mm_hash}." + ) assert pos_info.is_embed is None, ( "Expected all positions to be contiguous and embeddings." @@ -1726,7 +1743,22 @@ def profile_run( assert mm_budget is not None # TODO: handle encoder-decoder models once we support them. - if (encoder_budget := mm_budget.get_encoder_budget()) > 0: + encoder_budget = mm_budget.get_encoder_budget() + # Check if this is a minimal budget case (encoder_budget == 1) + # which occurs when all modalities are zero but enable_mm_embeds=True + if ( + encoder_budget == 1 + and mm_config is not None + and mm_config.enable_mm_embeds + and not mm_budget.max_tokens_by_modality + ): + logger.info( + "All modalities are disabled but enable_mm_embeds=True. " + "Initializing encoder cache with None values for memory " + "optimization." + ) + self.encoder_cache["dummy_none"] = None + elif encoder_budget > 0: # NOTE: Currently model is profiled with a single non-text # modality with the max possible input tokens even when # it supports multiple. diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 31ccf7f15746..b079fa32eb2b 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -54,6 +54,7 @@ def __init__( encoder_compute_budget, encoder_cache_size = compute_mm_encoder_budget( scheduler_config, max_tokens_by_modality, + model_config, ) self.encoder_compute_budget = encoder_compute_budget From 56f51e82ada74bbf181d47917d0ae2b2f5e4bddf Mon Sep 17 00:00:00 2001 From: Reagan Date: Mon, 29 Dec 2025 17:37:40 -0800 Subject: [PATCH 2/4] change --- tests/v1/core/test_encoder_cache_manager.py | 122 +++++++++++++++++++- vllm/v1/core/encoder_cache_manager.py | 17 ++- vllm/v1/core/sched/scheduler.py | 18 ++- 3 files changed, 149 insertions(+), 8 deletions(-) diff --git a/tests/v1/core/test_encoder_cache_manager.py b/tests/v1/core/test_encoder_cache_manager.py index 511ff48c401c..d6b4c6a2fb92 100644 --- a/tests/v1/core/test_encoder_cache_manager.py +++ b/tests/v1/core/test_encoder_cache_manager.py @@ -4,7 +4,10 @@ import torch from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange -from vllm.v1.core.encoder_cache_manager import EncoderCacheManager +from vllm.v1.core.encoder_cache_manager import ( + EncoderCacheManager, + EncoderDecoderCacheManager, +) pytestmark = pytest.mark.cpu_test @@ -247,3 +250,120 @@ def get_num_encoder_embeds(self, input_id: int) -> int: assert num_embeds_before == 0 assert num_embeds_in_range == 2 + +def test_embedding_only_mode_budget_bypass(): + manager = EncoderCacheManager(cache_size=1000, enable_mm_embeds=True) + req = MockRequest("req1", ["img1"], [256]) + + assert manager.can_allocate( + req, 0, encoder_compute_budget=1, num_embeds_to_schedule=0 + ) + + manager.allocate(req, 0) + assert "img1" in manager.cached + assert manager.num_free_slots == 1000 - 256 + + +@pytest.mark.parametrize( + "budget,num_embeds,expected_result,description", + [ + # Insufficient budget: request exceeds compute budget + (100, 256, False, "budget insufficient: 256 > 100"), + # Sufficient budget: request fits within compute budget + (300, 256, True, "budget sufficient: 256 <= 300"), + ], +) +def test_normal_mode_budget_enforcement( + budget, num_embeds, expected_result, description +): + manager = EncoderCacheManager(cache_size=1000, enable_mm_embeds=False) + req = MockRequest("req1", ["img1"], [num_embeds]) + + result = manager.can_allocate(req, 0, budget, 0) + assert result == expected_result, description + + +@pytest.mark.parametrize( + "enable_mm_embeds,budget,num_embeds,expected_result", + [ + # Case 1: enable_mm_embeds=False even with budget=1 + ( + False, + 1, + 10, + False, + ), + # Case 2: budget not minimal even with enable_mm_embeds=True + ( + True, + 100, + 256, + False, + ), + ], +) +def test_no_false_embedding_only_detection( + enable_mm_embeds, budget, num_embeds, expected_result +): + cache_size = 1000 if enable_mm_embeds else 100 + manager = EncoderCacheManager( + cache_size=cache_size, enable_mm_embeds=enable_mm_embeds + ) + req = MockRequest("req1", ["img1"], [num_embeds]) + + result = manager.can_allocate(req, 0, budget, 0) + assert result == expected_result + + +def test_embedding_only_mode_cache_space_limits(): + manager = EncoderCacheManager(cache_size=100, enable_mm_embeds=True) + + req = MockRequest("req1", ["img1"], [256]) + + encoder_compute_budget = 1 + num_embeds_to_schedule = 0 + + assert not manager.can_allocate( + req, 0, encoder_compute_budget, num_embeds_to_schedule + ) + + manager2 = EncoderCacheManager(cache_size=300, enable_mm_embeds=True) + assert manager2.can_allocate(req, 0, encoder_compute_budget, num_embeds_to_schedule) + + +def test_encoder_decoder_embedding_only_mode(): + manager = EncoderDecoderCacheManager(cache_size=500, enable_mm_embeds=True) + + req = MockRequest("req1", ["emb1"], [256]) + + encoder_compute_budget = 1 + num_embeds_to_schedule = 0 + + assert manager.can_allocate(req, 0, encoder_compute_budget, num_embeds_to_schedule) + + manager.allocate(req, 0) + assert manager.num_free_slots == 500 - 256 + + +def test_embedding_only_full_workflow(): + manager = EncoderCacheManager(cache_size=1000, enable_mm_embeds=True) + + req1 = MockRequest("req1", ["img1", "img2"], [256, 256]) + req2 = MockRequest("req2", ["img3"], [256]) + + encoder_compute_budget = 1 + + assert manager.can_allocate(req1, 0, encoder_compute_budget, 0) + manager.allocate(req1, 0) + + assert manager.can_allocate(req1, 1, encoder_compute_budget, 256) + manager.allocate(req1, 1) + + assert manager.can_allocate(req2, 0, encoder_compute_budget, 512) + manager.allocate(req2, 0) + + assert manager.num_free_slots == 1000 - 768 + assert len(manager.cached) == 3 + + manager.free(req1) + assert manager.num_freeable_slots == 1000 - 256 diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index c5f47ba2d9ca..bc3696a9a364 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -65,10 +65,11 @@ class EncoderCacheManager: last call to get_freed_mm_hashes(). This list is cleared on return. """ - def __init__(self, cache_size: int): + def __init__(self, cache_size: int, enable_mm_embeds: bool = False): self.cache_size = cache_size self.num_free_slots = cache_size self.num_freeable_slots = cache_size + self.enable_mm_embeds = enable_mm_embeds # mm_hash of mm_data => ids of requests that reference the mm_data self.cached: dict[str, set[str]] = {} @@ -142,8 +143,10 @@ def can_allocate( """ num_embeds = request.get_num_encoder_embeds(input_id) + is_embedding_only_mode = self.enable_mm_embeds and encoder_compute_budget == 1 + # Not enough compute budget - if num_embeds > encoder_compute_budget: + if not is_embedding_only_mode and num_embeds > encoder_compute_budget: return False num_embeds += num_embeds_to_schedule @@ -371,9 +374,10 @@ def compute_mm_encoder_budget( # utilize the cache and this class will fold into EncoderCacheManager, as # differences with MM models shrink. class EncoderDecoderCacheManager(EncoderCacheManager): - def __init__(self, cache_size: int): + def __init__(self, cache_size: int, enable_mm_embeds: bool = False): self.cache_size = cache_size self.num_free_slots = cache_size + self.enable_mm_embeds = enable_mm_embeds self.freed: list[str] = [] def check_and_update_cache(self, request: Request, input_id: int) -> bool: @@ -387,9 +391,12 @@ def can_allocate( num_embeds_to_schedule: int, ) -> bool: num_encoder_embeds = request.get_num_encoder_embeds(input_id) + + is_embedding_only_mode = self.enable_mm_embeds and encoder_compute_budget == 1 + # Not enough compute budget - if num_encoder_embeds > encoder_compute_budget: - return False + if not is_embedding_only_mode and num_encoder_embeds > encoder_compute_budget: + return False num_encoder_embeds += num_embeds_to_schedule # Enough free slots diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index da8339558b14..c629b61c469d 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -176,6 +176,14 @@ def __init__( mm_registry=mm_registry, ) + # Extract enable_mm_embeds flag for embedding-only mode detection + enable_mm_embeds = False + if ( + vllm_config.model_config is not None + and vllm_config.model_config.multimodal_config is not None + ): + enable_mm_embeds = vllm_config.model_config.multimodal_config.enable_mm_embeds + # NOTE(woosuk): Here, "encoder" includes the vision encoder (and # projector if needed) for MM models as well as encoder-decoder # transformers. @@ -184,9 +192,15 @@ def __init__( # the encoder cache will not be initialized because cache size is 0 # for these models. self.encoder_cache_manager = ( - EncoderDecoderCacheManager(cache_size=encoder_cache_size) + EncoderDecoderCacheManager( + cache_size=encoder_cache_size, + enable_mm_embeds=enable_mm_embeds, + ) if self.is_encoder_decoder - else EncoderCacheManager(cache_size=encoder_cache_size) + else EncoderCacheManager( + cache_size=encoder_cache_size, + enable_mm_embeds=enable_mm_embeds, + ) ) # For encoder-decoder models, allocate the maximum number of tokens for Cross # Attn blocks, as for Whisper its input is always padded to the maximum length. From bab74fc4f50ba1d777c501be3714573b551378c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CReagan?= <“reaganjlee@gmail.com”> Date: Wed, 31 Dec 2025 00:05:47 +0000 Subject: [PATCH 3/4] WIP --- example_embedding_only_curl.py | 79 ++++++++++ example_embedding_only_serve.py | 123 +++++++++++++++ test_embedding_only_fix.py | 185 +++++++++++++++++++++++ vllm/entrypoints/chat_utils.py | 8 +- vllm/model_executor/models/qwen2_5_vl.py | 28 +++- vllm/model_executor/models/qwen2_vl.py | 9 +- vllm/multimodal/processing.py | 151 ++++++++++++++++-- 7 files changed, 569 insertions(+), 14 deletions(-) create mode 100644 example_embedding_only_curl.py create mode 100644 example_embedding_only_serve.py create mode 100755 test_embedding_only_fix.py diff --git a/example_embedding_only_curl.py b/example_embedding_only_curl.py new file mode 100644 index 000000000000..863a4c121855 --- /dev/null +++ b/example_embedding_only_curl.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Helper script to generate a curl command for testing embedding-only mode. + +This script creates an example embedding and outputs a curl command that you can run. +""" + +import json +import torch +from vllm.utils.serial_utils import tensor2base64 + +def create_example_image_embedding( + num_images: int = 1, + feature_size: int = 256, + hidden_size: int = 4096, + dtype: torch.dtype = torch.float16 +) -> torch.Tensor: + """Create a dummy image embedding tensor.""" + embedding = torch.randn( + num_images, + feature_size, + hidden_size, + dtype=dtype + ) + return embedding + +def main(): + # Create example embedding + image_embeds = create_example_image_embedding( + num_images=1, + feature_size=256, + hidden_size=4096, + dtype=torch.float16 + ) + + # Convert to base64 + base64_image_embedding = tensor2base64(image_embeds) + + # Create the request payload + payload = { + "model": "Qwen/Qwen2.5-VL-3B-Instruct", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What do you see in this image? Describe it briefly." + }, + { + "type": "image_embeds", + "image_embeds": base64_image_embedding, + }, + ], + }, + ], + "max_tokens": 100, + "temperature": 0.7, + } + + # Print curl command + print("="*70) + print("CURL COMMAND:") + print("="*70) + print("\ncurl -X POST http://localhost:8000/v1/chat/completions \\") + print(" -H 'Content-Type: application/json' \\") + print(" -d '{}'".format(json.dumps(payload, indent=2))) + print("\n" + "="*70) + print("\nNote: The embedding is included in the JSON payload above.") + print("Embedding shape:", image_embeds.shape) + print("="*70) + +if __name__ == "__main__": + main() + diff --git a/example_embedding_only_serve.py b/example_embedding_only_serve.py new file mode 100644 index 000000000000..c4f45ee276d4 --- /dev/null +++ b/example_embedding_only_serve.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +""" +Example: Running vLLM in embedding-only mode (text mode) with embedding inputs + +This example demonstrates: +1. How to start the server with embedding-only mode for memory savings +2. How to create and send embedding inputs via the OpenAI API +""" + +import json +import torch +from openai import OpenAI +from vllm.utils.serial_utils import tensor2base64 + +# ============================================================================ +# STEP 1: Start the server with this command: +# ============================================================================ +""" +vllm serve Qwen/Qwen2.5-VL-7B-Instruct \ + --enable-mm-embeds \ + --limit-mm-per-prompt '{"image": 0, "video": 0}' \ + --port 8000 +""" + +# ============================================================================ +# STEP 2: Create a client and example embedding +# ============================================================================ + +def create_example_image_embedding( + num_images: int = 1, + feature_size: int = 256, # Typical image feature size + hidden_size: int = 4096, # Hidden size for Qwen2.5-VL models + dtype: torch.dtype = torch.float16 +) -> torch.Tensor: + """ + Create a dummy image embedding tensor. + + Shape: (num_images, feature_size, hidden_size) + For Qwen2.5-VL models: (1, 256, 4096) + """ + # Create random embeddings (in practice, these would come from your encoder) + embedding = torch.randn( + num_images, + feature_size, + hidden_size, + dtype=dtype + ) + return embedding + + +def main(): + # Initialize OpenAI client + client = OpenAI( + api_key="EMPTY", # Not needed for local server + base_url="http://localhost:8000/v1", + ) + + # Get the model name from the server (must match the model the server was started with) + try: + models = client.models.list() + if models.data: + model_name = models.data[0].id + print(f"Using model from server: {model_name}") + else: + raise ValueError("No models found on server") + except Exception as e: + print(f"Warning: Could not get model from server: {e}") + print("Using default model name. Make sure it matches your server.") + model_name = "Qwen/Qwen2.5-VL-7B-Instruct" # Fallback: must match server + + # Create example image embedding + # Shape: (num_images, feature_size, hidden_size) + # For Qwen2.5-VL models, typical shape is (1, 256, 4096) + print("Creating example image embedding...") + image_embeds = create_example_image_embedding( + num_images=1, + feature_size=256, + hidden_size=4096, + dtype=torch.float16 + ) + print(f"Embedding shape: {image_embeds.shape}") + + # Convert to base64 for API transmission + base64_image_embedding = tensor2base64(image_embeds) + print("Embedding serialized to base64") + + # Send request with embedding input + print("\nSending chat completion request...") + chat_completion = client.chat.completions.create( + model=model_name, + messages=[ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What do you see in this image? Describe it briefly." + }, + { + "type": "image_embeds", + "image_embeds": base64_image_embedding, + }, + ], + }, + ], + max_tokens=100, + temperature=0.7, + ) + + print("\n" + "="*60) + print("Response:") + print("="*60) + print(chat_completion.choices[0].message.content) + print("="*60) + + +if __name__ == "__main__": + main() + diff --git a/test_embedding_only_fix.py b/test_embedding_only_fix.py new file mode 100755 index 000000000000..7227a2a654fe --- /dev/null +++ b/test_embedding_only_fix.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +""" +Combined test script: Starts server and runs embedding-only test +This allows agentic access to test the fix in one command +""" + +import subprocess +import time +import sys +import signal +import os +import threading +from pathlib import Path + +try: + import requests +except ImportError: + print("Installing requests...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "requests", "-q"]) + import requests + +# Import the test client +from example_embedding_only_serve import main as run_client_test +from openai import OpenAI + +MODEL = "Qwen/Qwen2.5-VL-7B-Instruct" +PORT = 8000 +SERVER_URL = f"http://localhost:{PORT}" + +def check_server_ready(max_wait=300, process=None): + """Check if server is ready by polling /health endpoint""" + print(f"Waiting up to {max_wait} seconds for server to be ready...") + for i in range(max_wait): + # Check if process is still alive + if process and process.poll() is not None: + print(f"\n✗ Server process died unexpectedly (exit code: {process.returncode})") + # Try to read any output + try: + output = process.stdout.read() + if output: + print("Server output:") + print(output[-2000:]) # Last 2000 chars + except: + pass + return False + + try: + response = requests.get(f"{SERVER_URL}/health", timeout=2) + if response.status_code == 200: + print(f"✓ Server is ready (took {i+1}s)") + return True + except: + pass + + # Show progress every 10 seconds + if (i + 1) % 10 == 0: + print(f" Still waiting... ({i+1}s/{max_wait}s)") + + time.sleep(1) + + print(f"\n✗ Server failed to start within {max_wait} seconds") + if process: + # Try to read any output + try: + output = process.stdout.read() + if output: + print("\nLast server output:") + print(output[-2000:]) # Last 2000 chars + except: + pass + return False + +def start_server(): + """Start vLLM server in background""" + print("Starting vLLM server...") + print(f"Model: {MODEL}") + print(f"Port: {PORT}") + + # Use a clean HuggingFace cache to avoid corrupted cache issues + import os + env = os.environ.copy() + clean_cache_dir = "/tmp/hf_cache_clean" + os.makedirs(clean_cache_dir, exist_ok=True) + env["HF_HOME"] = clean_cache_dir + print(f"Using HuggingFace cache: {clean_cache_dir}") + + cmd = [ + "python", "-m", "vllm.entrypoints.openai.api_server", + "--model", MODEL, + "--enable-mm-embeds", + "--limit-mm-per-prompt", '{"image": 0, "video": 0}', + "--port", str(PORT), + "--max-model-len", "2048", # Smaller for faster startup + ] + + # Start server process - write to both file and capture + log_file = open("/tmp/vllm_server.log", "w") + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=env + ) + + print(f"Server process started (PID: {process.pid})") + print("Server logs are being written to /tmp/vllm_server.log") + print("You can monitor progress with: tail -f /tmp/vllm_server.log") + + # Start a thread to copy output to log file + def log_output(): + try: + for line in process.stdout: + log_file.write(line) + log_file.flush() + # Also print important lines + if any(keyword in line.lower() for keyword in ["error", "exception", "ready", "uvicorn", "started"]): + print(f" [SERVER] {line.rstrip()}") + except: + pass + + log_thread = threading.Thread(target=log_output, daemon=True) + log_thread.start() + + if not check_server_ready(max_wait=300, process=process): + print("✗ Server failed to start within timeout") + log_file.close() + process.terminate() + return None + + log_file.close() + return process + +def run_test(): + """Run the client test""" + print("\n" + "="*60) + print("Running client test...") + print("="*60 + "\n") + + try: + run_client_test() + print("\n✓ Test completed successfully!") + return True + except Exception as e: + print(f"\n✗ Test failed: {e}") + import traceback + traceback.print_exc() + return False + +def main(): + """Main test runner""" + server_process = None + + try: + # Start server + server_process = start_server() + if not server_process: + print("Failed to start server") + return 1 + + # Run test + success = run_test() + + return 0 if success else 1 + + except KeyboardInterrupt: + print("\n\nInterrupted by user") + return 130 + finally: + # Cleanup: kill server + if server_process: + print("\nShutting down server...") + try: + server_process.terminate() + server_process.wait(timeout=10) + except subprocess.TimeoutExpired: + print("Force killing server...") + server_process.kill() + server_process.wait() + print("Server stopped") + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index ab055dfb1fb0..af3cf185a729 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -729,10 +729,16 @@ def add( An optional uuid can be added which serves as a unique identifier of the media. """ + is_embedding = modality.endswith("_embeds") input_modality = modality.replace("_embeds", "") num_items = len(self._items_by_modality[modality]) + 1 - self.mm_processor.validate_num_items(input_modality, num_items) + # validate_num_items now handles embedding detection internally + self.mm_processor.validate_num_items( + input_modality, + num_items, + is_embedding=is_embedding + ) self._items_by_modality[modality].append(item) self._uuids_by_modality[modality].append(uuid) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index b730ac031589..b8ab7a4d866d 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -972,6 +972,24 @@ def _get_prompt_updates( merge_length = image_processor.merge_size**2 def get_replacement_qwen2vl(item_idx: int, modality: str): + # #region agent log + import json + with open('/workspace/vllm/.cursor/debug.log', 'a') as f: + f.write(json.dumps({ + "sessionId": "debug-session", + "runId": "run1", + "hypothesisId": "E", + "location": "qwen2_5_vl.py:974", + "message": "get_replacement_qwen2vl entry", + "data": { + "modality": modality, + "item_idx": item_idx, + "out_mm_kwargs_keys": list(out_mm_kwargs.keys()), + "mm_items_keys": list(mm_items.keys()) + }, + "timestamp": __import__('time').time() * 1000 + }) + '\n') + # #endregion out_item = out_mm_kwargs[modality][item_idx] grid_thw = out_item[f"{modality}_grid_thw"].data assert isinstance(grid_thw, torch.Tensor) @@ -998,13 +1016,21 @@ def get_replacement_qwen2vl(item_idx: int, modality: str): return [placeholder[modality]] * num_tokens + # Only create prompt updates for modalities that are present in BOTH mm_items AND out_mm_kwargs + # out_mm_kwargs may be empty for embeddings (passthrough data), so we need to check both + available_modalities = [ + modality for modality in ("image", "video") + if modality in mm_items and modality in out_mm_kwargs + ] + + return [ PromptReplacement( modality=modality, target=[placeholder[modality]], replacement=partial(get_replacement_qwen2vl, modality=modality), ) - for modality in ("image", "video") + for modality in available_modalities ] diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 321fbd764c0f..7d72a1138f4a 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -1095,13 +1095,20 @@ def get_replacement_qwen2vl(item_idx: int, modality: str): num_tokens = int(grid_thw.prod()) // merge_length return [placeholder[modality]] * num_tokens + # Only create prompt updates for modalities that are present in BOTH mm_items AND out_mm_kwargs + # out_mm_kwargs may be empty for embeddings (passthrough data), so we need to check both + available_modalities = [ + modality for modality in ("image", "video") + if modality in mm_items and modality in out_mm_kwargs + ] + return [ PromptReplacement( modality=modality, target=[placeholder[modality]], replacement=partial(get_replacement_qwen2vl, modality=modality), ) - for modality in ("image", "video") + for modality in available_modalities ] def _get_mm_fields_config( diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing.py index 057b4951f9ad..9872553ca1e6 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing.py @@ -1344,7 +1344,35 @@ def validate_num_items( self, modality: str, num_items: int, + mm_items: MultiModalDataItems | None = None, + is_embedding: bool = False, ) -> None: + """ + Validate that the number of items for a modality doesn't exceed limits. + + Args: + modality: The modality name (e.g., "image", "video") + num_items: The number of items to validate + mm_items: Optional MultiModalDataItems to check if this is an embedding. + If provided and the modality contains embeddings, validation + is skipped when enable_mm_embeds=True. + is_embedding: If True, indicates this is an embedding modality (used + when mm_items is not available, e.g., in chat_utils). + """ + mm_config = self.info.ctx.model_config.get_multimodal_config() + + # Check if this is an embedding modality + if mm_config.enable_mm_embeds: + # Check via is_embedding parameter (for chat_utils where mm_items isn't available) + if is_embedding: + return + + # Check via mm_items (for processing.py where mm_items is available) + if mm_items is not None and modality in mm_items: + modality_items = mm_items[modality] + if isinstance(modality_items, (EmbeddingItems, DictEmbeddingItems)): + return + supported_limit = self.supported_mm_limits.get(modality, 0) allowed_limit = self.allowed_mm_limits.get(modality, 0) @@ -1352,7 +1380,7 @@ def validate_num_items( supported_limit = allowed_limit limit = min(supported_limit, allowed_limit) - + if num_items > limit: msg = f"At most {limit} {modality}(s) may be provided in one prompt." @@ -1374,6 +1402,23 @@ def _to_mm_items( """ mm_items = self.data_parser.parse_mm_data(mm_data) + # #region agent log + import json + with open('/workspace/vllm/.cursor/debug.log', 'a') as f: + f.write(json.dumps({ + "sessionId": "debug-session", + "runId": "run1", + "hypothesisId": "D", + "location": "processing.py:1440", + "message": "After parse_mm_data - checking mm_items", + "data": { + "mm_items_keys": list(mm_items.keys()), + "mm_items_dict": {k: type(v).__name__ for k, v in mm_items.items()}, + "mm_data_keys": list(mm_data.keys()) if mm_data else [] + }, + "timestamp": __import__('time').time() * 1000 + }) + '\n') + # #endregion mm_config = self.info.ctx.model_config.get_multimodal_config() if not mm_config.enable_mm_embeds: for modality, items in mm_items.items(): @@ -1384,14 +1429,24 @@ def _to_mm_items( ) for modality, items in mm_items.items(): - # When enable_mm_embeds=True, allow embedding inputs even if - # modality limits are zero (for memory optimization) - if mm_config.enable_mm_embeds and isinstance( - items, (EmbeddingItems, DictEmbeddingItems) - ): - continue - self.validate_num_items(modality, len(items)) - + # validate_num_items now handles embedding detection internally + self.validate_num_items(modality, len(items), mm_items=mm_items) + + # #region agent log + with open('/workspace/vllm/.cursor/debug.log', 'a') as f: + f.write(json.dumps({ + "sessionId": "debug-session", + "runId": "run1", + "hypothesisId": "D", + "location": "processing.py:1460", + "message": "After validation - final mm_items", + "data": { + "mm_items_keys": list(mm_items.keys()), + "mm_items_dict": {k: type(v).__name__ for k, v in mm_items.items()} + }, + "timestamp": __import__('time').time() * 1000 + }) + '\n') + # #endregion return mm_items @abstractmethod @@ -2035,10 +2090,39 @@ def _validate_mm_kwargs( self, mm_kwargs: MultiModalKwargsOptionalItems, mm_item_counts: Mapping[str, int], + mm_items: MultiModalDataItems | None = None, ) -> None: + mm_config = self.info.ctx.model_config.get_multimodal_config() + for modality, item_count in mm_item_counts.items(): items = mm_kwargs.get(modality, []) + # Skip validation for embedding modalities when enable_mm_embeds=True + # Embeddings are passthrough data and don't appear in mm_kwargs + if mm_config.enable_mm_embeds and mm_items is not None: + if modality in mm_items: + modality_items = mm_items[modality] + if isinstance(modality_items, (EmbeddingItems, DictEmbeddingItems)): + # #region agent log + import json + with open('/workspace/vllm/.cursor/debug.log', 'a') as f: + f.write(json.dumps({ + "sessionId": "debug-session", + "runId": "run1", + "hypothesisId": "F", + "location": "processing.py:2140", + "message": "Skipping mm_kwargs validation for embedding modality", + "data": { + "modality": modality, + "item_count": item_count, + "mm_kwargs_items_len": len(items), + "is_embedding": True + }, + "timestamp": __import__('time').time() * 1000 + }) + '\n') + # #endregion + continue + if len(items) != item_count: raise RuntimeError( f"Expected there to be {item_count} {modality} items in " @@ -2054,10 +2138,39 @@ def _validate_mm_updates( self, mm_updates: MultiModalPromptUpdates, mm_item_counts: Mapping[str, int], + mm_items: MultiModalDataItems | None = None, ) -> None: + mm_config = self.info.ctx.model_config.get_multimodal_config() + for modality, item_count in mm_item_counts.items(): placeholders = mm_updates.get(modality, []) + # Skip validation for embedding modalities when enable_mm_embeds=True + # Embeddings don't require prompt updates (they're passthrough data) + if mm_config.enable_mm_embeds and mm_items is not None: + if modality in mm_items: + modality_items = mm_items[modality] + if isinstance(modality_items, (EmbeddingItems, DictEmbeddingItems)): + # #region agent log + import json + with open('/workspace/vllm/.cursor/debug.log', 'a') as f: + f.write(json.dumps({ + "sessionId": "debug-session", + "runId": "run1", + "hypothesisId": "G", + "location": "processing.py:2160", + "message": "Skipping mm_updates validation for embedding modality", + "data": { + "modality": modality, + "item_count": item_count, + "placeholders_len": len(placeholders), + "is_embedding": True + }, + "timestamp": __import__('time').time() * 1000 + }) + '\n') + # #endregion + continue + if len(placeholders) != item_count: raise RuntimeError( f"Expected there to be {item_count} prompt updates " @@ -2094,9 +2207,25 @@ def _maybe_apply_prompt_updates( mm_prompt_updates: MultiModalPromptUpdates, is_update_applied: bool, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + # #region agent log + import json + with open('/workspace/vllm/.cursor/debug.log', 'a') as f: + f.write(json.dumps({ + "sessionId": "debug-session", + "runId": "run1", + "hypothesisId": "D", + "location": "processing.py:2162", + "message": "Before get_all_counts - checking mm_items", + "data": { + "mm_items_keys": list(mm_items.keys()), + "mm_items_dict": {k: type(v).__name__ for k, v in mm_items.items()} + }, + "timestamp": __import__('time').time() * 1000 + }) + '\n') + # #endregion mm_item_counts = mm_items.get_all_counts() - self._validate_mm_kwargs(mm_kwargs, mm_item_counts) - self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + self._validate_mm_kwargs(mm_kwargs, mm_item_counts, mm_items=mm_items) + self._validate_mm_updates(mm_prompt_updates, mm_item_counts, mm_items=mm_items) if is_update_applied: mm_placeholders = self._find_mm_placeholders( From 8f5669f93d4f73f22bfa25f7683b7b7260976598 Mon Sep 17 00:00:00 2001 From: User Date: Sun, 4 Jan 2026 08:20:39 +0000 Subject: [PATCH 4/4] change --- test_embedding_only_fix.py | 8 ++------ vllm/multimodal/processing.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/test_embedding_only_fix.py b/test_embedding_only_fix.py index 7227a2a654fe..e99f8f0c2a53 100755 --- a/test_embedding_only_fix.py +++ b/test_embedding_only_fix.py @@ -76,16 +76,12 @@ def start_server(): print(f"Model: {MODEL}") print(f"Port: {PORT}") - # Use a clean HuggingFace cache to avoid corrupted cache issues + # Use existing environment (including default HuggingFace cache) import os env = os.environ.copy() - clean_cache_dir = "/tmp/hf_cache_clean" - os.makedirs(clean_cache_dir, exist_ok=True) - env["HF_HOME"] = clean_cache_dir - print(f"Using HuggingFace cache: {clean_cache_dir}") cmd = [ - "python", "-m", "vllm.entrypoints.openai.api_server", + sys.executable, "-m", "vllm.entrypoints.openai.api_server", "--model", MODEL, "--enable-mm-embeds", "--limit-mm-per-prompt", '{"image": 0, "video": 0}', diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing.py index 9872553ca1e6..493116a74398 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing.py @@ -1500,15 +1500,28 @@ def _get_mm_prompt_updates( hf_processor_mm_kwargs: Mapping[str, object], out_mm_kwargs: MultiModalKwargsItems, ) -> MultiModalPromptUpdates: + # Filter out embedding modalities - they don't need prompt updates + # This prevents KeyError in model's _get_prompt_updates when accessing + # out_mm_kwargs[modality] for embeddings (HF processor doesn't produce + # output for embedding modalities) + mm_config = self.info.ctx.model_config.get_multimodal_config() + if mm_config.enable_mm_embeds: + filtered_mm_items = MultiModalDataItems({ + k: v for k, v in mm_items.items() + if not isinstance(v, (EmbeddingItems, DictEmbeddingItems)) + }) + else: + filtered_mm_items = mm_items + unbound_prompt_updates = self._get_prompt_updates( - mm_items=mm_items, + mm_items=filtered_mm_items, hf_processor_mm_kwargs=hf_processor_mm_kwargs, out_mm_kwargs=out_mm_kwargs, ) mm_prompt_updates = self._bind_and_group_updates( unbound_prompt_updates, - mm_items.get_all_counts(), + filtered_mm_items.get_all_counts(), ) for modality, prompt_updates in mm_prompt_updates.items(): @@ -2186,10 +2199,21 @@ def _validate_mm_placeholders( self, mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]], mm_item_counts: Mapping[str, int], + mm_items: MultiModalDataItems | None = None, ) -> None: + mm_config = self.info.ctx.model_config.get_multimodal_config() + for modality, item_count in mm_item_counts.items(): placeholders = mm_placeholders.get(modality, []) + # Skip validation for embedding modalities when enable_mm_embeds=True + # Embeddings don't require prompt placeholders (they're passthrough data) + if mm_config.enable_mm_embeds and mm_items is not None: + if modality in mm_items: + modality_items = mm_items[modality] + if isinstance(modality_items, (EmbeddingItems, DictEmbeddingItems)): + continue + if len(placeholders) != item_count: raise RuntimeError( f"Expected there to be {item_count} prompt placeholders " @@ -2232,13 +2256,13 @@ def _maybe_apply_prompt_updates( prompt_ids, mm_prompt_updates, ) - self._validate_mm_placeholders(mm_placeholders, mm_item_counts) + self._validate_mm_placeholders(mm_placeholders, mm_item_counts, mm_items=mm_items) else: prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids, mm_prompt_updates, ) - self._validate_mm_placeholders(mm_placeholders, mm_item_counts) + self._validate_mm_placeholders(mm_placeholders, mm_item_counts, mm_items=mm_items) return prompt_ids, mm_placeholders