From 9b49ff8edc44551addc397f5796c993af8aa1d79 Mon Sep 17 00:00:00 2001 From: YunzhuLu Date: Sun, 12 Jul 2026 21:06:03 +0800 Subject: [PATCH 1/3] [Bugfix][MM] Fix MiniCPM-V Transformers v5 compatibility Signed-off-by: YunzhuLu --- tests/models/registry.py | 7 -- vllm/model_executor/models/minicpmv.py | 160 ++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 10 deletions(-) diff --git a/tests/models/registry.py b/tests/models/registry.py index 41850ad8fbbb..fc24af4e9325 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1084,13 +1084,6 @@ def check_available_online( "4.0": "openbmb/MiniCPM-V-4", "4.5": "openbmb/MiniCPM-V-4_5", }, - max_transformers_version="4.57", - transformers_version_reason={ - "vllm": ( - "MiniCPMVBatchFeature is incompatible with its base class in " - "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78" - ) - }, trust_remote_code=True, ), "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo( diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 60b111298214..975966f9c27b 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -37,7 +37,8 @@ from torch import nn from torch.nn.init import trunc_normal_ from transformers import BatchFeature, PretrainedConfig -from typing_extensions import TypeVar +from transformers.dynamic_module_utils import get_class_from_dynamic_module +from typing_extensions import TypeVar, assert_never from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -74,14 +75,25 @@ from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, + MultiModalPromptUpdates, + MultiModalPromptUpdatesApplyResult, + PlaceholderFeaturesInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ResolvedPromptUpdate, + UpdateMode, + _all_items_found, + _find_matches, _seq2text, + _seq2tokens, ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.processor import ( + _merge_mm_kwargs, + cached_get_image_processor, +) from vllm.utils.collection_utils import flatten_2d_lists from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import set_default_torch_dtype @@ -98,6 +110,9 @@ # For profile run _MAX_FRAMES_PER_VIDEO = 16 +# Same class name in every MiniCPM-V repo +_MINICPMV_IMAGE_PROCESSOR_CLASS_REF = "image_processing_minicpmv.MiniCPMVImageProcessor" + class MiniCPMVImagePixelInputs(TensorSchema): """ @@ -545,12 +560,33 @@ def get_hf_config(self): return self.ctx.get_hf_config() def get_hf_processor(self, **kwargs: object): + cached = getattr(self, "_minicpmv_hf_processor", None) + if cached is not None: + return cached + + # AutoProcessor only for tokenizer; its image_processor is resolved by + # class name and can pick the wrong checkpoint across MiniCPM-V versions. hf_processor = self.ctx.get_hf_processor(**kwargs) + model_config = self.ctx.model_config + processor_cls = get_class_from_dynamic_module( + _MINICPMV_IMAGE_PROCESSOR_CLASS_REF, + model_config.model, + revision=model_config.revision, + trust_remote_code=model_config.trust_remote_code, + ) + image_processor = cached_get_image_processor( + model_config.model, + revision=model_config.revision, + trust_remote_code=model_config.trust_remote_code, + processor_cls_overrides=processor_cls, + **_merge_mm_kwargs(model_config, processor_cls, **kwargs), + ) + from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor vendored_processor = MiniCPMVProcessor( - image_processor=hf_processor.image_processor, + image_processor=image_processor, tokenizer=hf_processor.tokenizer, version=self.get_model_version(), ) @@ -558,13 +594,13 @@ def get_hf_processor(self, **kwargs: object): # NumPy arrays are considered as Iterable but not Sequence in # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428 - image_processor = hf_processor.image_processor # type: ignore # transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std` for attr in ("mean", "std", "image_mean", "image_std"): val = getattr(image_processor, attr, None) if isinstance(val, np.ndarray): setattr(image_processor, attr, val.tolist()) + self._minicpmv_hf_processor = hf_processor return hf_processor def get_image_processor(self, **kwargs: object): @@ -844,6 +880,124 @@ def process_mm_inputs( **self.process_videos(mm_data, mm_kwargs, tok_kwargs), } + def _apply_prompt_updates( + self, + token_ids: list[int], + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + """Apply multi-modal prompt updates to token IDs.""" + tokenizer = self.info.get_tokenizer() + + new_token_ids, match_result = self._apply_token_matches( + token_ids, + mm_prompt_updates, + ) + + # If the search text does not represent a special token, + # it may have different token IDs in the prompt, because + # the tokens may go across the boundaries of the search text. + # ---- + # e.g. when searching for "foo" in "food", if "food" itself makes + # up a token, then the token ID of "foo" will not appear at all + # ---- + # Since it is inefficient to search for all possible tokenizations + # of the search text in the prompt, we instead perform string-based + # updates on the decoded token IDs, then encode them back. + if not all( + all(update_idx is not None for update_idx in update_idxs) + for update_idxs in match_result.values() + ): + new_token_ids, match_result = self._apply_prompt_updates_by_text_locate( + _seq2text(tokenizer, token_ids, use_cache=False), + mm_prompt_updates, + ) + + matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list) + for modality, update_idxs in match_result.items(): + for item_idx, update_idx in enumerate(update_idxs): + assert update_idx is not None, ( + "Failed to apply prompt replacement for " + f"mm_items[{modality!r}][{item_idx}]" + ) + + matched_updates[modality].append( + [mm_prompt_updates[modality][item_idx][update_idx]] + ) + + placeholders = self._find_mm_placeholders( + new_token_ids, + dict(matched_updates), + ) + + return new_token_ids, placeholders + + def _apply_prompt_updates_by_text_locate( + self, + text: str, + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]: + tokenizer = self.info.get_tokenizer() + + mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()} + + out_seqs = list[list[int]]() + out_result: MultiModalPromptUpdatesApplyResult = { + m: [None] * len(items) for m, items in mm_prompt_updates.items() + } + + # Early exit if no items to find + mm_found_counts = { + m: sum(r is not None for r in res) for m, res in out_result.items() + } + if _all_items_found(mm_item_counts, mm_found_counts): + return _seq2tokens(tokenizer, text), out_result + + prev_end_idx = 0 + while True: + mode, matches_to_apply = _find_matches( + text, + mm_prompt_updates, + tokenizer, + prev_end_idx=prev_end_idx, + current_result=out_result, + ) + + if mode is None: + break # No more matches to find + + for (modality, item_idx), (match, update_idx) in matches_to_apply: + matched_update = mm_prompt_updates[modality][item_idx][update_idx] + matched_content = matched_update.content.full + + if mode == UpdateMode.INSERT: + end_idx_to_insert = match.end_idx + elif mode == UpdateMode.REPLACE: + end_idx_to_insert = match.start_idx + else: + assert_never(mode) + + out_seqs.append( + _seq2tokens( + tokenizer, text[prev_end_idx:end_idx_to_insert], use_cache=False + ) + ) + out_seqs.append(_seq2tokens(tokenizer, matched_content)) + out_result[modality][item_idx] = update_idx + + # Exclude overlapping matches + prev_end_idx = match.end_idx + + # Early exit if all items found + mm_found_counts = { + m: sum(r is not None for r in res) for m, res in out_result.items() + } + if _all_items_found(mm_item_counts, mm_found_counts): + break + + out_seqs.append(_seq2tokens(tokenizer, text[prev_end_idx:], use_cache=False)) + + return flatten_2d_lists(out_seqs), out_result + def _base_call_hf_processor( self, prompts: list[str], From 48163223af4eaf21f077f64c88c20bfa5600d2e5 Mon Sep 17 00:00:00 2001 From: YunzhuLu Date: Wed, 15 Jul 2026 09:01:43 +0800 Subject: [PATCH 2/3] fix processor cache implement Signed-off-by: YunzhuLu --- .../multimodal/processing/test_minicpmv.py | 41 +++++++++++++++++++ vllm/model_executor/models/minicpmv.py | 37 +++++++++++------ 2 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 tests/models/multimodal/processing/test_minicpmv.py diff --git a/tests/models/multimodal/processing/test_minicpmv.py b/tests/models/multimodal/processing/test_minicpmv.py new file mode 100644 index 000000000000..bad8afc6661e --- /dev/null +++ b/tests/models/multimodal/processing/test_minicpmv.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Tests for MiniCPMV's multimodal preprocessing.""" + +import pytest + +from vllm.multimodal import MULTIMODAL_REGISTRY + +from ...utils import build_model_context + + +@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) +def test_get_hf_processor_for_different_kwargs(model_id: str): + """Calls with different kwargs must not reuse stale processor instances.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + info = processor.info + processor_1 = info.get_hf_processor(max_slice_nums=1) + processor_2 = info.get_hf_processor(max_slice_nums=2) + assert processor_1.image_processor.max_slice_nums == 1 + assert processor_2.image_processor.max_slice_nums == 2 + assert processor_1 is not processor_2 + + +@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) +def test_get_hf_processor_for_same_kwargs(model_id: str): + """Same kwargs should return the cached processor instance.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + info = processor.info + processor_a = info.get_hf_processor(max_slice_nums=1) + processor_b = info.get_hf_processor(max_slice_nums=1) + assert processor_a is processor_b + assert processor_a.image_processor.max_slice_nums == 1 diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 975966f9c27b..a33784b655e7 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -27,7 +27,7 @@ import math from collections import defaultdict from collections.abc import Callable, Iterable, Mapping, Sequence -from functools import partial +from functools import cached_property, partial from itertools import chain from typing import Annotated, Any, Literal, TypeAlias @@ -556,31 +556,42 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): image_pattern = "(./)" video_pattern = "()" + @cached_property + def _image_processor_cls(self): + model_config = self.ctx.model_config + return get_class_from_dynamic_module( + _MINICPMV_IMAGE_PROCESSOR_CLASS_REF, + model_config.model, + revision=model_config.revision, + trust_remote_code=model_config.trust_remote_code, + ) + def get_hf_config(self): return self.ctx.get_hf_config() def get_hf_processor(self, **kwargs: object): - cached = getattr(self, "_minicpmv_hf_processor", None) - if cached is not None: - return cached + model_config = self.ctx.model_config + processor_cls = self._image_processor_cls + merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs) + + cache_key = tuple(sorted(merged_kwargs.items())) + cached_processors = getattr(self, "_minicpmv_hf_processor_cache", None) + if cached_processors is None: + cached_processors = {} + self._minicpmv_hf_processor_cache = cached_processors + if cache_key in cached_processors: + return cached_processors[cache_key] # AutoProcessor only for tokenizer; its image_processor is resolved by # class name and can pick the wrong checkpoint across MiniCPM-V versions. hf_processor = self.ctx.get_hf_processor(**kwargs) - model_config = self.ctx.model_config - processor_cls = get_class_from_dynamic_module( - _MINICPMV_IMAGE_PROCESSOR_CLASS_REF, - model_config.model, - revision=model_config.revision, - trust_remote_code=model_config.trust_remote_code, - ) image_processor = cached_get_image_processor( model_config.model, revision=model_config.revision, trust_remote_code=model_config.trust_remote_code, processor_cls_overrides=processor_cls, - **_merge_mm_kwargs(model_config, processor_cls, **kwargs), + **merged_kwargs, ) from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor @@ -600,7 +611,7 @@ def get_hf_processor(self, **kwargs: object): if isinstance(val, np.ndarray): setattr(image_processor, attr, val.tolist()) - self._minicpmv_hf_processor = hf_processor + cached_processors[cache_key] = hf_processor return hf_processor def get_image_processor(self, **kwargs: object): From d45da6ee5df82ed54d04f1d0b76778b8ed1ffa89 Mon Sep 17 00:00:00 2001 From: YunzhuLu Date: Fri, 24 Jul 2026 15:25:41 +0800 Subject: [PATCH 3/3] address review comments Signed-off-by: YunzhuLu --- .../multimodal/processing/test_minicpmv.py | 51 ++++++-- vllm/model_executor/models/minicpmv.py | 121 +++++------------- vllm/multimodal/processing/processor.py | 33 +++++ 3 files changed, 107 insertions(+), 98 deletions(-) diff --git a/tests/models/multimodal/processing/test_minicpmv.py b/tests/models/multimodal/processing/test_minicpmv.py index bad8afc6661e..9424ea39595c 100644 --- a/tests/models/multimodal/processing/test_minicpmv.py +++ b/tests/models/multimodal/processing/test_minicpmv.py @@ -3,6 +3,7 @@ """Tests for MiniCPMV's multimodal preprocessing.""" +import numpy as np import pytest from vllm.multimodal import MULTIMODAL_REGISTRY @@ -11,7 +12,7 @@ @pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) -def test_get_hf_processor_for_different_kwargs(model_id: str): +def test_get_hf_processor_for_same_model_different_kwargs(model_id: str): """Calls with different kwargs must not reuse stale processor instances.""" ctx = build_model_context( model_id, @@ -23,19 +24,51 @@ def test_get_hf_processor_for_different_kwargs(model_id: str): processor_2 = info.get_hf_processor(max_slice_nums=2) assert processor_1.image_processor.max_slice_nums == 1 assert processor_2.image_processor.max_slice_nums == 2 - assert processor_1 is not processor_2 + + +@pytest.mark.parametrize( + "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")] +) +def test_image_processor_for_dif_model(model_ids): + model_id_25, model_id_4 = model_ids + + ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1}) + processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config) + image_processor_25 = processor_25.info.get_image_processor() + + ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1}) + processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config) + image_processor_4 = processor_4.info.get_image_processor() + + assert type(image_processor_25) is not type(image_processor_4) + assert type(image_processor_25).__module__ != type(image_processor_4).__module__ @pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"]) -def test_get_hf_processor_for_same_kwargs(model_id: str): - """Same kwargs should return the cached processor instance.""" +def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str): ctx = build_model_context( model_id, limit_mm_per_prompt={"image": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - info = processor.info - processor_a = info.get_hf_processor(max_slice_nums=1) - processor_b = info.get_hf_processor(max_slice_nums=1) - assert processor_a is processor_b - assert processor_a.image_processor.max_slice_nums == 1 + tokenizer = ctx.get_tokenizer() + + messages = [ + {"role": "user", "content": "(./)\nWhat is in this image?"} + ] + prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + ) + image = np.zeros((768, 1024, 3), dtype=np.uint8) + + mm_items = processor.info.parse_mm_data({"image": [image]}) + processed = processor( + prompt, + mm_items=mm_items, + hf_processor_mm_kwargs={}, + ) + image_placeholders = processed["mm_placeholders"].get("image", []) + assert len(image_placeholders) == 1 + assert image_placeholders[0].length > 0 diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index a33784b655e7..7aba0a29b054 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -37,8 +37,11 @@ from torch import nn from torch.nn.init import trunc_normal_ from transformers import BatchFeature, PretrainedConfig -from transformers.dynamic_module_utils import get_class_from_dynamic_module -from typing_extensions import TypeVar, assert_never +from transformers.dynamic_module_utils import ( + get_class_from_dynamic_module, + resolve_trust_remote_code, +) +from typing_extensions import TypeVar from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -76,17 +79,12 @@ BaseMultiModalProcessor, BaseProcessingInfo, MultiModalPromptUpdates, - MultiModalPromptUpdatesApplyResult, PlaceholderFeaturesInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ResolvedPromptUpdate, - UpdateMode, - _all_items_found, - _find_matches, _seq2text, - _seq2tokens, ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -94,6 +92,7 @@ _merge_mm_kwargs, cached_get_image_processor, ) +from vllm.transformers_utils.utils import convert_model_repo_to_path from vllm.utils.collection_utils import flatten_2d_lists from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import set_default_torch_dtype @@ -110,9 +109,6 @@ # For profile run _MAX_FRAMES_PER_VIDEO = 16 -# Same class name in every MiniCPM-V repo -_MINICPMV_IMAGE_PROCESSOR_CLASS_REF = "image_processing_minicpmv.MiniCPMVImageProcessor" - class MiniCPMVImagePixelInputs(TensorSchema): """ @@ -559,11 +555,34 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo): @cached_property def _image_processor_cls(self): model_config = self.ctx.model_config + model_path = convert_model_repo_to_path(model_config.model) + + from transformers import ImageProcessingMixin + + image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict( + model_path, + revision=model_config.revision, + token=model_config.hf_token, + ) + + auto_map = image_processor_config.get("auto_map") or {} + class_ref = auto_map.get("AutoImageProcessor") + if not class_ref: + raise ValueError( + "Missing auto_map['AutoImageProcessor'] in image processor config " + f"for {model_config.model!r}" + ) + + resolve_trust_remote_code( + model_config.trust_remote_code, + model_config.model, + has_local_code=False, + has_remote_code=True, + ) return get_class_from_dynamic_module( - _MINICPMV_IMAGE_PROCESSOR_CLASS_REF, + class_ref, model_config.model, revision=model_config.revision, - trust_remote_code=model_config.trust_remote_code, ) def get_hf_config(self): @@ -574,14 +593,6 @@ def get_hf_processor(self, **kwargs: object): processor_cls = self._image_processor_cls merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs) - cache_key = tuple(sorted(merged_kwargs.items())) - cached_processors = getattr(self, "_minicpmv_hf_processor_cache", None) - if cached_processors is None: - cached_processors = {} - self._minicpmv_hf_processor_cache = cached_processors - if cache_key in cached_processors: - return cached_processors[cache_key] - # AutoProcessor only for tokenizer; its image_processor is resolved by # class name and can pick the wrong checkpoint across MiniCPM-V versions. hf_processor = self.ctx.get_hf_processor(**kwargs) @@ -611,7 +622,6 @@ def get_hf_processor(self, **kwargs: object): if isinstance(val, np.ndarray): setattr(image_processor, attr, val.tolist()) - cached_processors[cache_key] = hf_processor return hf_processor def get_image_processor(self, **kwargs: object): @@ -918,7 +928,7 @@ def _apply_prompt_updates( all(update_idx is not None for update_idx in update_idxs) for update_idxs in match_result.values() ): - new_token_ids, match_result = self._apply_prompt_updates_by_text_locate( + new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens( _seq2text(tokenizer, token_ids, use_cache=False), mm_prompt_updates, ) @@ -942,73 +952,6 @@ def _apply_prompt_updates( return new_token_ids, placeholders - def _apply_prompt_updates_by_text_locate( - self, - text: str, - mm_prompt_updates: MultiModalPromptUpdates, - ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]: - tokenizer = self.info.get_tokenizer() - - mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()} - - out_seqs = list[list[int]]() - out_result: MultiModalPromptUpdatesApplyResult = { - m: [None] * len(items) for m, items in mm_prompt_updates.items() - } - - # Early exit if no items to find - mm_found_counts = { - m: sum(r is not None for r in res) for m, res in out_result.items() - } - if _all_items_found(mm_item_counts, mm_found_counts): - return _seq2tokens(tokenizer, text), out_result - - prev_end_idx = 0 - while True: - mode, matches_to_apply = _find_matches( - text, - mm_prompt_updates, - tokenizer, - prev_end_idx=prev_end_idx, - current_result=out_result, - ) - - if mode is None: - break # No more matches to find - - for (modality, item_idx), (match, update_idx) in matches_to_apply: - matched_update = mm_prompt_updates[modality][item_idx][update_idx] - matched_content = matched_update.content.full - - if mode == UpdateMode.INSERT: - end_idx_to_insert = match.end_idx - elif mode == UpdateMode.REPLACE: - end_idx_to_insert = match.start_idx - else: - assert_never(mode) - - out_seqs.append( - _seq2tokens( - tokenizer, text[prev_end_idx:end_idx_to_insert], use_cache=False - ) - ) - out_seqs.append(_seq2tokens(tokenizer, matched_content)) - out_result[modality][item_idx] = update_idx - - # Exclude overlapping matches - prev_end_idx = match.end_idx - - # Early exit if all items found - mm_found_counts = { - m: sum(r is not None for r in res) for m, res in out_result.items() - } - if _all_items_found(mm_item_counts, mm_found_counts): - break - - out_seqs.append(_seq2tokens(tokenizer, text[prev_end_idx:], use_cache=False)) - - return flatten_2d_lists(out_seqs), out_result - def _base_call_hf_processor( self, prompts: list[str], diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 7b24cd3fcb5e..b27ffccbc265 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -862,6 +862,29 @@ def apply_text_matches( return "".join(texts), result +def apply_text_matches_as_segmented_tokens( + prompt: str, + mm_prompt_updates: "MultiModalPromptUpdates", + tokenizer: TokenizerLike | None, +) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]: + """ + Apply the updates in `mm_prompt_updates` to `prompt`. + + Matches are exclusive even when multiple modalities share + the same placeholder tokens. In that case, the modality that + appears earlier in `mm_prompt_updates` takes priority. + + Each segment is encoded separately instead of being joined into one + string and encoded in a single pass. Joining first would let BPE merge + tokens across a segment boundary, silently change how a text + (non-special-token) placeholder is tokenized. + """ + texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer) + token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts] + + return flatten_2d_lists(token_id_seqs), result + + def _iter_placeholders( prompt: list[int], mm_prompt_updates: "MultiModalPromptUpdates", @@ -1525,6 +1548,16 @@ def _apply_text_matches( tokenizer = self.info.get_tokenizer() return apply_text_matches(prompt, mm_prompt_updates, tokenizer) + def _apply_text_matches_as_segmented_tokens( + self, + prompt: str, + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]: + tokenizer = self.info.get_tokenizer() + return apply_text_matches_as_segmented_tokens( + prompt, mm_prompt_updates, tokenizer + ) + def _apply_prompt_updates( self, token_ids: list[int],