diff --git a/tests/ut/models/test_deepseek_v4_vision_preprocess.py b/tests/ut/models/test_deepseek_v4_vision_preprocess.py index 9a4b7d8e782f..44674c4043b8 100644 --- a/tests/ut/models/test_deepseek_v4_vision_preprocess.py +++ b/tests/ut/models/test_deepseek_v4_vision_preprocess.py @@ -1,11 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Ascend project +import hashlib import threading import time from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace +import numpy as np +import pytest import torch from PIL import Image from transformers import BatchFeature @@ -85,6 +88,155 @@ def test_local_image_processor_builds_vit_and_llm_inputs(): assert output["perm"].numel() == output["llm_grid"].prod().item() +def _pattern_image(width: int, height: int) -> Image.Image: + y, x = np.indices((height, width), dtype=np.uint16) + pixels = np.stack( + ( + (x * 3 + y * 5) % 256, + (x * 7 + y * 11 + 17) % 256, + (x * 13 + y * 19 + 29) % 256, + ), + axis=-1, + ).astype(np.uint8) + return Image.fromarray(pixels, "RGB") + + +def _tensor_sha256(tensor: torch.Tensor) -> str: + data = tensor.contiguous().view(torch.uint8).numpy().tobytes() + return hashlib.sha256(data).hexdigest() + + +@pytest.mark.parametrize( + ("name", "width", "height", "expected"), + [ + ( + "square", + 392, + 392, + ( + (784, 3, 14, 14), + "cb9ebeb276de6b449bc942dd4d55398a2f9fbdbfdd9a28b4cea93be8cdb1847c", + (28, 28), + (10, 10), + 114, + "da4b32a5c728550fcc6f88567ef03c3952aad1bba7d8f2d8ee222ec89103c556", + 100, + "f376025fbf067fb7d845127be7b72bf0fafdc6a3510bd7b658caca8c5cd198d5", + ), + ), + ( + "panorama", + 1600, + 100, + ( + (780, 3, 14, 14), + "2539b7bf59517689ec0b2f3e1417e5544ef617dde14ebbb88366222376d65cb0", + (10, 78), + (4, 26), + 110, + "bd33de4bbf3afc05780806c320b7928f6bf774b86d6c6840a054aedfa9dc6a61", + 104, + "a433d2bfca12832470ca925ad0403a82553a08abae9f159afacd062a1a0e3313", + ), + ), + ( + "portrait", + 100, + 1600, + ( + (920, 3, 14, 14), + "9e22cfa3743e92e091cd86a26774175f216616aa706751c22765fa70486bdd9c", + (115, 8), + (39, 3), + 162, + "74c67b7ea557942f339c7fac659f7756c13e6e8f76d7f50f267b62bb086f8b26", + 117, + "cfae26c65300c4e315e973c22678c59372b3903a01a7a70a453bd72b990038fb", + ), + ), + ( + "min_pixel", + 32, + 24, + ( + (768, 3, 14, 14), + "3e159e37db91587c2a7ed16cd4bc9b9e12c9e9ba3cebaf919dc97d4a943bd8ac", + (24, 32), + (8, 11), + 98, + "084c2845dd3f9aa822817b17bb532621559b5d6aaa640ab612c02613e546f7a5", + 88, + "27af7da74d9a95489b32bcdc31c4420995f81bbfe0724cd3c3bdaa40d3759e4d", + ), + ), + ( + "max_token", + 4096, + 3072, + ( + (2961, 3, 14, 14), + "ccff1f42208ce479433d8ebbe31abadddfae727e1602d0284907d651e399af17", + (47, 63), + (16, 21), + 354, + "24890234f78a3b081027729729127c131480224863bec0741864c5f7778dbc55", + 336, + "053baba523007d1d2884962752a3df17ede2320a19c2221321c33ed38f895311", + ), + ), + ], +) +def test_processor_matches_official_image_golden(name, width, height, expected): + config = SimpleNamespace( + vision_patch_size=14, + vision_downsample_ratio=3, + vision_max_n_token=384, + vision_min_pixels=147456, + vision_max_wh_ratio=8, + ) + output = DeepseekV4VLProcessor(config)(images=[_pattern_image(width, height)]) + ( + patch_shape, + patch_hash, + vit_grid, + llm_grid, + types_size, + types_hash, + perm_size, + perm_hash, + ) = expected + + assert tuple(output["patches"].shape) == patch_shape, name + assert _tensor_sha256(output["patches"]) == patch_hash + assert output["vit_grid"].tolist() == [list(vit_grid)] + assert output["llm_grid"].tolist() == [list(llm_grid)] + assert output["types"].numel() == types_size + assert _tensor_sha256(output["types"]) == types_hash + assert output["perm"].numel() == perm_size + assert _tensor_sha256(output["perm"]) == perm_hash + + +@pytest.mark.parametrize( + ("start_pos", "expected_size", "expected_hash"), + [ + (0, 17, "79bbacd708dc7bb1a333454394bd8c3d7f8ceb34b09d034d7e5804be7e69b29f"), + (1, 16, "f28cdc513680e49b6aa4a842f8ceabc2cf26d4f0e609b1d32ce9a62641f5a59f"), + (2, 15, "ccae3d732e7f5df568cf40aeeb02ad044e006b4a4d265b53f1427f3fca4f0cae"), + (3, 14, "fa1822de9eda236f7b38ebad58011b0009f335f39b295ea282f4f35cf29e3919"), + ], +) +def test_image_block_matches_official_position_golden( + start_pos, + expected_size, + expected_hash, +): + types, perm = prep.build_image_block(3, 2, start_pos) + + assert types.numel() == expected_size + assert _tensor_sha256(types) == expected_hash + assert _tensor_sha256(perm) == ("9ce675ac27d3af2951b2da39d6c3dc65c4205a27bf5ecc20ab33d1d7387c7412") + + def test_v027_prompt_updates_add_position_dependent_compress_pad(): base = prep.IMAGE_SENTINEL_BASE_ID image_token_id = 7 diff --git a/tests/ut/patch/platform/test_deepseek_v4_vision.py b/tests/ut/patch/platform/test_deepseek_v4_vision.py new file mode 100644 index 000000000000..430594e750cb --- /dev/null +++ b/tests/ut/patch/platform/test_deepseek_v4_vision.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +from vllm.entrypoints import chat_utils +from vllm.entrypoints.chat_utils import MODALITY_PLACEHOLDERS_MAP + +from vllm_ascend.patch.platform.patch_deepseek_v4_vision import ( + _make_multimodal_parser_patch, +) + + +def _tracker(*, vision: bool): + return SimpleNamespace( + model_config=SimpleNamespace( + hf_config=SimpleNamespace( + model_type="deepseek_v4" if vision else "qwen3", + vision_n_layers=32 if vision else 0, + ) + ) + ) + + +class _ImageContentParser: + def __init__(self, model_config): + self.model_config = model_config + self._images = [] + + def parse_image(self, image_url, uuid=None): + del image_url, uuid + self._images.append("<｜deepseek_image｜>") + + def mm_placeholder_storage(self): + return {MODALITY_PLACEHOLDERS_MAP["image"]: self._images} + + +class _ImageTracker: + def __init__(self): + self.model_config = SimpleNamespace( + hf_config=SimpleNamespace( + model_type="deepseek_v4", + vision_n_layers=32, + ), + enable_prompt_embeds=False, + ) + + def create_parser(self, mm_processor_kwargs=None): + del mm_processor_kwargs + return _ImageContentParser(self.model_config) + + +def test_vision_parser_preserves_content_order_and_separator(): + captured: dict[str, object] = {} + + def original( + role, + parts, + mm_tracker, + *, + wrap_dicts, + interleave_strings, + mm_processor_kwargs=None, + multimodal_content_part_separator="\n", + ): + captured.update( + role=role, + parts=parts, + mm_tracker=mm_tracker, + wrap_dicts=wrap_dicts, + interleave_strings=interleave_strings, + mm_processor_kwargs=mm_processor_kwargs, + multimodal_content_part_separator=multimodal_content_part_separator, + ) + return ["parsed"] + + patched = _make_multimodal_parser_patch(original) + result = patched( + "user", + ["before", {"type": "image_url"}, "after"], + _tracker(vision=True), + wrap_dicts=False, + interleave_strings=False, + ) + + assert result == ["parsed"] + assert captured["interleave_strings"] is True + assert captured["multimodal_content_part_separator"] == "\n\n" + + parsed = chat_utils._parse_chat_message_content_parts( + "user", + [ + {"type": "text", "text": "before"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + }, + {"type": "text", "text": "after"}, + ], + _ImageTracker(), + wrap_dicts=False, + interleave_strings=False, + ) + assert parsed == [ + { + "role": "user", + "content": "before\n\n<｜deepseek_image｜>\n\nafter", + } + ] + + parsed = chat_utils._parse_chat_message_content_parts( + "user", + [ + {"type": "text", "text": "before\ninside"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + }, + {"type": "text", "text": "after"}, + ], + _ImageTracker(), + wrap_dicts=False, + interleave_strings=False, + ) + assert parsed[0]["content"] == "before\ninside\n\n<｜deepseek_image｜>\n\nafter" + + +def test_non_vision_parser_arguments_are_unchanged(): + captured: dict[str, object] = {} + + def original(role, parts, mm_tracker, *, wrap_dicts, interleave_strings): + captured.update( + role=role, + parts=parts, + mm_tracker=mm_tracker, + wrap_dicts=wrap_dicts, + interleave_strings=interleave_strings, + ) + return [] + + patched = _make_multimodal_parser_patch(original) + patched( + "user", + ["text"], + _tracker(vision=False), + wrap_dicts=True, + interleave_strings=False, + ) + + assert captured["interleave_strings"] is False + assert captured["wrap_dicts"] is True + + prompt = chat_utils._get_full_multimodal_text_prompt( + {"": [""]}, + ["before", "", "after"], + interleave_strings=True, + ) + assert prompt == "before\n\nafter" diff --git a/tests/ut/worker/test_model_runner_v1.py b/tests/ut/worker/test_model_runner_v1.py index a22b262e3500..7b34b98bec9b 100644 --- a/tests/ut/worker/test_model_runner_v1.py +++ b/tests/ut/worker/test_model_runner_v1.py @@ -30,6 +30,56 @@ from vllm_ascend.worker.model_runner_v1 import NPUModelRunner +class TestRawTokenMultimodalCompileSelection(unittest.TestCase): + def test_cached_image_prefill_skips_text_only_compiled_graph(self): + runner = NPUModelRunner.__new__(NPUModelRunner) + runner.model_config = SimpleNamespace(is_encoder_decoder=False, requires_raw_input_tokens=True) + image = SimpleNamespace(mm_position=SimpleNamespace(offset=81, length=344)) + runner.requests = {"image": SimpleNamespace(num_computed_tokens=0, mm_features=[image])} + for encoder_inputs in ({}, {"image": [0]}): + for start, count, expected in ( + (0, 452, True), + (0, 81, False), + (81, 1, True), + (200, 32, True), + (424, 1, True), + (425, 27, False), + (452, 1, False), + ): + with self.subTest(encoder_inputs=encoder_inputs, start=start, count=count): + runner.requests["image"].num_computed_tokens = start + output = SimpleNamespace( + scheduled_encoder_inputs=encoder_inputs, + num_scheduled_tokens={"image": count}, + ) + self.assertEqual(runner._should_skip_compiled_for_mm(output), expected) + + def test_text_decode_and_non_raw_models_keep_compiled_path(self): + runner = NPUModelRunner.__new__(NPUModelRunner) + runner.model_config = SimpleNamespace(is_encoder_decoder=False, requires_raw_input_tokens=True) + runner.requests = {"text": SimpleNamespace(num_computed_tokens=0, mm_features=[])} + output = SimpleNamespace(scheduled_encoder_inputs={}, num_scheduled_tokens={"text": 100}) + self.assertFalse(runner._should_skip_compiled_for_mm(output)) + runner.model_config.requires_raw_input_tokens = False + output.scheduled_encoder_inputs = {"text": [0]} + self.assertFalse(runner._should_skip_compiled_for_mm(output)) + runner.model_config.is_encoder_decoder = True + self.assertTrue(runner._should_skip_compiled_for_mm(output)) + + def test_mixed_decode_and_cached_image_batch(self): + runner = NPUModelRunner.__new__(NPUModelRunner) + runner.model_config = SimpleNamespace(is_encoder_decoder=False, requires_raw_input_tokens=True) + runner.requests = { + "text": SimpleNamespace(num_computed_tokens=100, mm_features=[]), + "image": SimpleNamespace( + num_computed_tokens=0, + mm_features=[SimpleNamespace(mm_position=SimpleNamespace(offset=81, length=344))], + ), + } + output = SimpleNamespace(scheduled_encoder_inputs={}, num_scheduled_tokens={"text": 1, "image": 452}) + self.assertTrue(runner._should_skip_compiled_for_mm(output)) + + class TestDummyRunSlotInvalidation(unittest.TestCase): def test_backend_metadata_sees_invalidated_dummy_slots(self): runner = NPUModelRunner.__new__(NPUModelRunner) diff --git a/vllm_ascend/patch/__init__.py b/vllm_ascend/patch/__init__.py index 780f2da6f4e7..957a20ce7327 100644 --- a/vllm_ascend/patch/__init__.py +++ b/vllm_ascend/patch/__init__.py @@ -51,24 +51,31 @@ # ** 2. File: platform/patch_deepseek_v4_vision.py** # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # 1. `vllm.transformers_utils.model_arch_config_convertor.MODEL_ARCH_CONFIG_CONVERTORS` +# 2. `vllm.entrypoints.chat_utils._parse_chat_message_content_parts` +# `vllm.entrypoints.chat_utils._get_full_multimodal_text_prompt` # Why: # The supported vLLM revision has the generic DeepSeek-V4 text config # conversion but does not identify a checkpoint with `vision_n_layers` # as the multimodal conditional-generation architecture. Without this # distinction, vllm-ascend cannot select its DeepSeek-V4 vision wrapper # or enable bidirectional attention over the image prefix. +# The generic string-content parser also moves multimodal placeholders +# ahead of text and joins content parts with one newline. The checkpoint +# renderer preserves image order and separates every part with two. # How: # Register an Ascend DeepSeek-V4 config conversion handler. For vision checkpoints # it selects `DeepseekV4ForConditionalGeneration`, enables multimodal # prefix-LM attention, and records the prefix-padding constraints used by -# the Ascend DSA path. Text-only DeepSeek-V4 behavior is unchanged. +# the Ascend DSA path. For vision checkpoints, preserve interleaved image +# placeholders and use the checkpoint's two-newline separator. Text-only +# DeepSeek-V4 and other model behavior is unchanged. # Related PR (if no, explain why): # https://github.com/vllm-project/vllm/pull/54566 # Future Plan: # Remove this patch once the supported vLLM revision natively maps # DeepSeek-V4 vision checkpoints to the conditional-generation model and -# exposes the required multimodal prefix-LM and padding metadata without -# replacing `MODEL_ARCH_CONFIG_CONVERTORS["deepseek_v4"]`. +# exposes both the required multimodal prefix-LM metadata and a +# model-specific content-part renderer without downstream patching. # # ** 3. File: platform/patch_distributed.py** # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/vllm_ascend/patch/platform/patch_deepseek_v4_vision.py b/vllm_ascend/patch/platform/patch_deepseek_v4_vision.py index f5db062e8ddd..98a04c696cde 100644 --- a/vllm_ascend/patch/platform/patch_deepseek_v4_vision.py +++ b/vllm_ascend/patch/platform/patch_deepseek_v4_vision.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Ascend project -"""vLLM v0.27 model-config compatibility for DeepSeek-V4 vision.""" +"""vLLM v0.27 compatibility for DeepSeek-V4 vision.""" +import inspect +from collections import Counter +from functools import wraps from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -10,6 +13,84 @@ _REGISTERED = False +def _is_deepseek_v4_vision_model(model_config: object) -> bool: + """Return whether a model config belongs to the DeepSeek-V4 vision path.""" + hf_config = getattr(model_config, "hf_config", None) + return getattr(hf_config, "model_type", None) == "deepseek_v4" and getattr(hf_config, "vision_n_layers", 0) > 0 + + +def _make_multimodal_parser_patch(original): + signature = inspect.signature(original) + + @wraps(original) + def patched(*args, **kwargs): + bound = signature.bind(*args, **kwargs) + bound.apply_defaults() + tracker = bound.arguments.get("mm_tracker") + if tracker is not None and _is_deepseek_v4_vision_model(tracker.model_config): + # The upstream encoding keeps image blocks in their input order and + # joins every content block with two newlines. + bound.arguments["interleave_strings"] = True + if "multimodal_content_part_separator" in signature.parameters: + bound.arguments["multimodal_content_part_separator"] = "\n\n" + return original(*bound.args, **bound.kwargs) + + patched.__dict__["_deepseek_v4_vision_parser_patch"] = True + return patched + + +def _make_multimodal_prompt_patch(original): + signature = inspect.signature(original) + + @wraps(original) + def patched(*args, **kwargs): + bound = signature.bind(*args, **kwargs) + bound.apply_defaults() + if not bound.arguments["interleave_strings"]: + return original(*bound.args, **bound.kwargs) + + separator = bound.arguments.get("multimodal_content_part_separator", "\n") + if separator != "\n\n": + return original(*bound.args, **bound.kwargs) + + # Let the v0.27 helper validate counts and consume interleaved + # placeholders, then rebuild from the original content-part strings. + # This keeps newlines inside a text part intact. + placeholder_counts = Counter( + placeholder + for placeholders in bound.arguments["placeholder_storage"].values() + for placeholder in placeholders + ) + bound.arguments["multimodal_content_part_separator"] = "\n" + original(*bound.args, **bound.kwargs) + texts = bound.arguments["texts"] + missing_placeholders = [] + for placeholder, count in placeholder_counts.items(): + remaining = count - sum(text.count(placeholder) for text in texts) + missing_placeholders.extend([placeholder] * remaining) + return separator.join(missing_placeholders + list(texts)) + + patched.__dict__["_deepseek_v4_vision_prompt_patch"] = True + return patched + + +def _patch_deepseek_v4_multimodal_parser() -> None: + """Preserve DeepSeek-V4 image block order in the v0.27 string parser.""" + try: + from vllm.entrypoints import chat_utils + + original = chat_utils._parse_chat_message_content_parts + original_prompt = chat_utils._get_full_multimodal_text_prompt + except (AttributeError, ImportError, TypeError): + return + + if not getattr(original_prompt, "_deepseek_v4_vision_prompt_patch", False): + chat_utils._get_full_multimodal_text_prompt = _make_multimodal_prompt_patch(original_prompt) + if getattr(original, "_deepseek_v4_vision_parser_patch", False): + return + chat_utils._parse_chat_message_content_parts = _make_multimodal_parser_patch(original) + + def register_deepseek_v4_vision_config_convertor() -> None: """Route vision checkpoints to the Ascend multimodal wrapper. @@ -18,6 +99,7 @@ def register_deepseek_v4_vision_config_convertor() -> None: ``model_arch_config_convertor`` itself is only partially initialized. """ global _REGISTERED + _patch_deepseek_v4_multimodal_parser() if _REGISTERED: return @@ -51,3 +133,6 @@ def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool: MODEL_ARCH_CONFIG_CONVERTORS["deepseek_v4"] = AscendDeepseekV4ModelArchConfigConvertor _REGISTERED = True + + +_patch_deepseek_v4_multimodal_parser() diff --git a/vllm_ascend/worker/model_runner_v1.py b/vllm_ascend/worker/model_runner_v1.py index 84e429f6ee80..731c964212cf 100644 --- a/vllm_ascend/worker/model_runner_v1.py +++ b/vllm_ascend/worker/model_runner_v1.py @@ -2390,15 +2390,7 @@ def execute_model( cudagraph_mode = CUDAGraphMode.NONE # Mark KV scales as calculated after the first forward pass self.calculate_kv_scales = False # type: ignore[has-type] - # Encoder-decoder models and raw-token multimodal models can only - # compile pure decode steps where no encoder inputs are present. The - # DeepSeek-V4 vision router needs raw sentinel ids during image - # prefill, so keep that pass eager. - num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs) - has_encoder_input = num_encoder_reqs > 0 and ( - self.model_config.is_encoder_decoder - or self.model_config.requires_raw_input_tokens - ) + skip_compiled = self._should_skip_compiled_for_mm(scheduler_output) # Run forward pass defer_kv_connector_finalize = self.speculative_config is not None and ( @@ -2417,7 +2409,7 @@ def execute_model( num_actual_tokens=scheduler_output.total_num_scheduled_tokens, model_instance=self.model, device_metadata_executor=active_device_metadata_executor, - skip_compiled=has_encoder_input, + skip_compiled=skip_compiled, has_sinks=self._has_sinks, eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ), @@ -2955,6 +2947,24 @@ def _update_full_graph_params_if_needed( self.speculative_config, ) + def _should_skip_compiled_for_mm(self, scheduler_output: "SchedulerOutput") -> bool: + if self.model_config.is_encoder_decoder and scheduler_output.scheduled_encoder_inputs: + return True + if not self.model_config.requires_raw_input_tokens: + return False + # The raw-token dummy run compiles with inputs_embeds=None. Image + # prefill must consume the merged embeddings instead, even when the + # encoder output is cached and no encoder execution is scheduled. + for req_id, num_tokens in scheduler_output.num_scheduled_tokens.items(): + state = self.requests[req_id] + start = state.num_computed_tokens + end = start + num_tokens + for feature in state.mm_features: + position = feature.mm_position + if start < position.offset + position.length and position.offset < end: + return True + return False + def _model_forward( self, num_tokens_padded: int,