[Feature][Model] Align DeepSeek V4 Vision inputs and cached-image execution

tests/ut/models/test_deepseek_v4_vision_preprocess.py

@@ -1,11 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Ascend project
 
+import hashlib
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor
 from types import SimpleNamespace
 
+import numpy as np
+import pytest
 import torch
 from PIL import Image
 from transformers import BatchFeature
@@ -85,6 +88,155 @@ def test_local_image_processor_builds_vit_and_llm_inputs():
     assert output["perm"].numel() == output["llm_grid"].prod().item()
 
 
+def _pattern_image(width: int, height: int) -> Image.Image:
+    y, x = np.indices((height, width), dtype=np.uint16)
+    pixels = np.stack(
+        (
+            (x * 3 + y * 5) % 256,
+            (x * 7 + y * 11 + 17) % 256,
+            (x * 13 + y * 19 + 29) % 256,
+        ),
+        axis=-1,
+    ).astype(np.uint8)
+    return Image.fromarray(pixels, "RGB")
+
+
+def _tensor_sha256(tensor: torch.Tensor) -> str:
+    data = tensor.contiguous().view(torch.uint8).numpy().tobytes()
+    return hashlib.sha256(data).hexdigest()
+
+
+@pytest.mark.parametrize(
+    ("name", "width", "height", "expected"),
+    [
+        (
+            "square",
+            392,
+            392,
+            (
+                (784, 3, 14, 14),
+                "cb9ebeb276de6b449bc942dd4d55398a2f9fbdbfdd9a28b4cea93be8cdb1847c",
+                (28, 28),
+                (10, 10),
+                114,
+                "da4b32a5c728550fcc6f88567ef03c3952aad1bba7d8f2d8ee222ec89103c556",
+                100,
+                "f376025fbf067fb7d845127be7b72bf0fafdc6a3510bd7b658caca8c5cd198d5",
+            ),
+        ),
+        (
+            "panorama",
+            1600,
+            100,
+            (
+                (780, 3, 14, 14),
+                "2539b7bf59517689ec0b2f3e1417e5544ef617dde14ebbb88366222376d65cb0",
+                (10, 78),
+                (4, 26),
+                110,
+                "bd33de4bbf3afc05780806c320b7928f6bf774b86d6c6840a054aedfa9dc6a61",
+                104,
+                "a433d2bfca12832470ca925ad0403a82553a08abae9f159afacd062a1a0e3313",
+            ),
+        ),
+        (
+            "portrait",
+            100,
+            1600,
+            (
+                (920, 3, 14, 14),
+                "9e22cfa3743e92e091cd86a26774175f216616aa706751c22765fa70486bdd9c",
+                (115, 8),
+                (39, 3),
+                162,
+                "74c67b7ea557942f339c7fac659f7756c13e6e8f76d7f50f267b62bb086f8b26",
+                117,
+                "cfae26c65300c4e315e973c22678c59372b3903a01a7a70a453bd72b990038fb",
+            ),
+        ),
+        (
+            "min_pixel",
+            32,
+            24,
+            (
+                (768, 3, 14, 14),
+                "3e159e37db91587c2a7ed16cd4bc9b9e12c9e9ba3cebaf919dc97d4a943bd8ac",
+                (24, 32),
+                (8, 11),
+                98,
+                "084c2845dd3f9aa822817b17bb532621559b5d6aaa640ab612c02613e546f7a5",
+                88,
+                "27af7da74d9a95489b32bcdc31c4420995f81bbfe0724cd3c3bdaa40d3759e4d",
+            ),
+        ),
+        (
+            "max_token",
+            4096,
+            3072,
+            (
+                (2961, 3, 14, 14),
+                "ccff1f42208ce479433d8ebbe31abadddfae727e1602d0284907d651e399af17",
+                (47, 63),
+                (16, 21),
+                354,
+                "24890234f78a3b081027729729127c131480224863bec0741864c5f7778dbc55",
+                336,
+                "053baba523007d1d2884962752a3df17ede2320a19c2221321c33ed38f895311",
+            ),
+        ),
+    ],
+)
+def test_processor_matches_official_image_golden(name, width, height, expected):
+    config = SimpleNamespace(
+        vision_patch_size=14,
+        vision_downsample_ratio=3,
+        vision_max_n_token=384,
+        vision_min_pixels=147456,
+        vision_max_wh_ratio=8,
+    )
+    output = DeepseekV4VLProcessor(config)(images=[_pattern_image(width, height)])
+    (
+        patch_shape,
+        patch_hash,
+        vit_grid,
+        llm_grid,
+        types_size,
+        types_hash,
+        perm_size,
+        perm_hash,
+    ) = expected
+
+    assert tuple(output["patches"].shape) == patch_shape, name
+    assert _tensor_sha256(output["patches"]) == patch_hash
+    assert output["vit_grid"].tolist() == [list(vit_grid)]
+    assert output["llm_grid"].tolist() == [list(llm_grid)]
+    assert output["types"].numel() == types_size
+    assert _tensor_sha256(output["types"]) == types_hash
+    assert output["perm"].numel() == perm_size
+    assert _tensor_sha256(output["perm"]) == perm_hash
+
+
+@pytest.mark.parametrize(
+    ("start_pos", "expected_size", "expected_hash"),
+    [
+        (0, 17, "79bbacd708dc7bb1a333454394bd8c3d7f8ceb34b09d034d7e5804be7e69b29f"),
+        (1, 16, "f28cdc513680e49b6aa4a842f8ceabc2cf26d4f0e609b1d32ce9a62641f5a59f"),
+        (2, 15, "ccae3d732e7f5df568cf40aeeb02ad044e006b4a4d265b53f1427f3fca4f0cae"),
+        (3, 14, "fa1822de9eda236f7b38ebad58011b0009f335f39b295ea282f4f35cf29e3919"),
+    ],
+)
+def test_image_block_matches_official_position_golden(
+    start_pos,
+    expected_size,
+    expected_hash,
+):
+    types, perm = prep.build_image_block(3, 2, start_pos)
+
+    assert types.numel() == expected_size
+    assert _tensor_sha256(types) == expected_hash
+    assert _tensor_sha256(perm) == ("9ce675ac27d3af2951b2da39d6c3dc65c4205a27bf5ecc20ab33d1d7387c7412")
+
+
 def test_v027_prompt_updates_add_position_dependent_compress_pad():
     base = prep.IMAGE_SENTINEL_BASE_ID
     image_token_id = 7

tests/ut/patch/platform/test_deepseek_v4_vision.py

@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from types import SimpleNamespace
+
+from vllm.entrypoints import chat_utils
+from vllm.entrypoints.chat_utils import MODALITY_PLACEHOLDERS_MAP
+
+from vllm_ascend.patch.platform.patch_deepseek_v4_vision import (
+    _make_multimodal_parser_patch,
+)
+
+
+def _tracker(*, vision: bool):
+    return SimpleNamespace(
+        model_config=SimpleNamespace(
+            hf_config=SimpleNamespace(
+                model_type="deepseek_v4" if vision else "qwen3",
+                vision_n_layers=32 if vision else 0,
+            )
+        )
+    )
+
+
+class _ImageContentParser:
+    def __init__(self, model_config):
+        self.model_config = model_config
+        self._images = []
+
+    def parse_image(self, image_url, uuid=None):
+        del image_url, uuid
+        self._images.append("<｜deepseek_image｜>")
+
+    def mm_placeholder_storage(self):
+        return {MODALITY_PLACEHOLDERS_MAP["image"]: self._images}
+
+
+class _ImageTracker:
+    def __init__(self):
+        self.model_config = SimpleNamespace(
+            hf_config=SimpleNamespace(
+                model_type="deepseek_v4",
+                vision_n_layers=32,
+            ),
+            enable_prompt_embeds=False,
+        )
+
+    def create_parser(self, mm_processor_kwargs=None):
+        del mm_processor_kwargs
+        return _ImageContentParser(self.model_config)
+
+
+def test_vision_parser_preserves_content_order_and_separator():
+    captured: dict[str, object] = {}
+
+    def original(
+        role,
+        parts,
+        mm_tracker,
+        *,
+        wrap_dicts,
+        interleave_strings,
+        mm_processor_kwargs=None,
+        multimodal_content_part_separator="\n",
+    ):
+        captured.update(
+            role=role,
+            parts=parts,
+            mm_tracker=mm_tracker,
+            wrap_dicts=wrap_dicts,
+            interleave_strings=interleave_strings,
+            mm_processor_kwargs=mm_processor_kwargs,
+            multimodal_content_part_separator=multimodal_content_part_separator,
+        )
+        return ["parsed"]
+
+    patched = _make_multimodal_parser_patch(original)
+    result = patched(
+        "user",
+        ["before", {"type": "image_url"}, "after"],
+        _tracker(vision=True),
+        wrap_dicts=False,
+        interleave_strings=False,
+    )
+
+    assert result == ["parsed"]
+    assert captured["interleave_strings"] is True
+    assert captured["multimodal_content_part_separator"] == "\n\n"
+
+    parsed = chat_utils._parse_chat_message_content_parts(
+        "user",
+        [
+            {"type": "text", "text": "before"},
+            {
+                "type": "image_url",
+                "image_url": {"url": "https://example.com/image.png"},
+            },
+            {"type": "text", "text": "after"},
+        ],
+        _ImageTracker(),
+        wrap_dicts=False,
+        interleave_strings=False,
+    )
+    assert parsed == [
+        {
+            "role": "user",
+            "content": "before\n\n<｜deepseek_image｜>\n\nafter",
+        }
+    ]
+
+    parsed = chat_utils._parse_chat_message_content_parts(
+        "user",
+        [
+            {"type": "text", "text": "before\ninside"},
+            {
+                "type": "image_url",
+                "image_url": {"url": "https://example.com/image.png"},
+            },
+            {"type": "text", "text": "after"},
+        ],
+        _ImageTracker(),
+        wrap_dicts=False,
+        interleave_strings=False,
+    )
+    assert parsed[0]["content"] == "before\ninside\n\n<｜deepseek_image｜>\n\nafter"
+
+
+def test_non_vision_parser_arguments_are_unchanged():
+    captured: dict[str, object] = {}
+
+    def original(role, parts, mm_tracker, *, wrap_dicts, interleave_strings):
+        captured.update(
+            role=role,
+            parts=parts,
+            mm_tracker=mm_tracker,
+            wrap_dicts=wrap_dicts,
+            interleave_strings=interleave_strings,
+        )
+        return []
+
+    patched = _make_multimodal_parser_patch(original)
+    patched(
+        "user",
+        ["text"],
+        _tracker(vision=False),
+        wrap_dicts=True,
+        interleave_strings=False,
+    )
+
+    assert captured["interleave_strings"] is False
+    assert captured["wrap_dicts"] is True
+
+    prompt = chat_utils._get_full_multimodal_text_prompt(
+        {"<image>": ["<image>"]},
+        ["before", "<image>", "after"],
+        interleave_strings=True,
+    )
+    assert prompt == "before\n<image>\nafter"

tests/ut/worker/test_model_runner_v1.py

@@ -30,6 +30,56 @@
 from vllm_ascend.worker.model_runner_v1 import NPUModelRunner
 
 
+class TestRawTokenMultimodalCompileSelection(unittest.TestCase):
+    def test_cached_image_prefill_skips_text_only_compiled_graph(self):
+        runner = NPUModelRunner.__new__(NPUModelRunner)
+        runner.model_config = SimpleNamespace(is_encoder_decoder=False, requires_raw_input_tokens=True)
+        image = SimpleNamespace(mm_position=SimpleNamespace(offset=81, length=344))
+        runner.requests = {"image": SimpleNamespace(num_computed_tokens=0, mm_features=[image])}
+        for encoder_inputs in ({}, {"image": [0]}):
+            for start, count, expected in (
+                (0, 452, True),
+                (0, 81, False),
+                (81, 1, True),
+                (200, 32, True),
+                (424, 1, True),
+                (425, 27, False),
+                (452, 1, False),
+            ):
+                with self.subTest(encoder_inputs=encoder_inputs, start=start, count=count):
+                    runner.requests["image"].num_computed_tokens = start
+                    output = SimpleNamespace(
+                        scheduled_encoder_inputs=encoder_inputs,
+                        num_scheduled_tokens={"image": count},
+                    )
+                    self.assertEqual(runner._should_skip_compiled_for_mm(output), expected)
+
+    def test_text_decode_and_non_raw_models_keep_compiled_path(self):
+        runner = NPUModelRunner.__new__(NPUModelRunner)
+        runner.model_config = SimpleNamespace(is_encoder_decoder=False, requires_raw_input_tokens=True)
+        runner.requests = {"text": SimpleNamespace(num_computed_tokens=0, mm_features=[])}
+        output = SimpleNamespace(scheduled_encoder_inputs={}, num_scheduled_tokens={"text": 100})
+        self.assertFalse(runner._should_skip_compiled_for_mm(output))
+        runner.model_config.requires_raw_input_tokens = False
+        output.scheduled_encoder_inputs = {"text": [0]}
+        self.assertFalse(runner._should_skip_compiled_for_mm(output))
+        runner.model_config.is_encoder_decoder = True
+        self.assertTrue(runner._should_skip_compiled_for_mm(output))
+
+    def test_mixed_decode_and_cached_image_batch(self):
+        runner = NPUModelRunner.__new__(NPUModelRunner)
+        runner.model_config = SimpleNamespace(is_encoder_decoder=False, requires_raw_input_tokens=True)
+        runner.requests = {
+            "text": SimpleNamespace(num_computed_tokens=100, mm_features=[]),
+            "image": SimpleNamespace(
+                num_computed_tokens=0,
+                mm_features=[SimpleNamespace(mm_position=SimpleNamespace(offset=81, length=344))],
+            ),
+        }
+        output = SimpleNamespace(scheduled_encoder_inputs={}, num_scheduled_tokens={"text": 1, "image": 452})
+        self.assertTrue(runner._should_skip_compiled_for_mm(output))
+
+
 class TestDummyRunSlotInvalidation(unittest.TestCase):
     def test_backend_metadata_sees_invalidated_dummy_slots(self):
         runner = NPUModelRunner.__new__(NPUModelRunner)