diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index fa2f76730d21..601fe9b8a819 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -17,6 +17,7 @@ PromptInsertion, PromptReplacement, _apply_matches, + _apply_token_matches_with_placeholders, apply_text_matches, apply_token_matches, find_mm_placeholders, @@ -567,171 +568,309 @@ def test_find_update_text( assert new_prompt == expected -@pytest.mark.parametrize( - ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501 - [ - # Tokenized test cases of `test_find_update_text` - # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf - ( - [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918], - { - # We use `` before `Image:` to test matches that - # occur out of order - "pattern_1": [32000], - "pattern_2": [9833, 28747], - "pattern_3": [918], +FIND_UPDATE_TOKENS_TEST_CASES = [ + # Tokenized test cases of `test_find_update_text` + # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf + ( + [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918], + { + # We use `` before `Image:` to test matches that + # occur out of order + "pattern_1": [32000], + "pattern_2": [9833, 28747], + "pattern_3": [918], + }, + { + # Test whether target is confused with replacement + "pattern_1": [32000, 32000], + # Test empty replacement + "pattern_2": [], + # Test dynamic replacement (beyond the form of `unit * count`) + "pattern_3": [1550, 918, 1550], + }, + { + PromptInsertion: { + 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918], + 1: [ + 1, + 9833, + 28747, + 32000, + 32000, + 32000, + 9833, + 28747, + 32000, + 32000, + 918, + 1550, + 918, + 1550, + ], # noqa: E501 + 2: [ + 1, + 9833, + 28747, + 32000, + 32000, + 32000, + 32000, + 32000, + 9833, + 28747, + 32000, + 32000, + 918, + 1550, + 918, + 1550, + 1550, + 918, + 1550, + ], # noqa: E501 }, - { - # Test whether target is confused with replacement - "pattern_1": [32000, 32000], - # Test empty replacement - "pattern_2": [], - # Test dynamic replacement (beyond the form of `unit * count`) - "pattern_3": [1550, 918, 1550], + PromptReplacement: { + 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918], + 1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501 + 2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550], }, - { - PromptInsertion: { - 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918], - 1: [ - 1, - 9833, - 28747, - 32000, - 32000, - 32000, - 9833, - 28747, - 32000, - 32000, - 918, - 1550, - 918, - 1550, - ], # noqa: E501 - 2: [ - 1, - 9833, - 28747, - 32000, - 32000, - 32000, - 32000, - 32000, - 9833, - 28747, - 32000, - 32000, - 918, - 1550, - 918, - 1550, - 1550, - 918, - 1550, - ], # noqa: E501 - }, - PromptReplacement: { - 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918], - 1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501 - 2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550], - }, + }, + ), + # Test index targets + ( + [], + { + "pattern_1": PromptIndexTargets.start(), + "pattern_2": PromptIndexTargets.prefix([32000]), + "pattern_3": PromptIndexTargets.end(), + }, + { + "pattern_1": [-1], + "pattern_2": [-2], + "pattern_3": [-3], + }, + { + PromptInsertion: { + 0: [], + 1: [-1, -3], + 2: [-1, -1, -3, -3], }, - ), - # Test index targets - ( - [], - { - "pattern_1": PromptIndexTargets.start(), - "pattern_2": PromptIndexTargets.prefix([32000]), - "pattern_3": PromptIndexTargets.end(), + PromptReplacement: { + 0: [], + 1: [-1, -3], + 2: [-1, -1, -3, -3], }, - { - "pattern_1": [-1], - "pattern_2": [-2], - "pattern_3": [-3], + }, + ), + ( + [32000], + { + "pattern_1": PromptIndexTargets.start(), + "pattern_2": PromptIndexTargets.prefix([32000]), + "pattern_3": PromptIndexTargets.end(), + }, + { + "pattern_1": [-1], + "pattern_2": [-2], + "pattern_3": [-3], + }, + { + PromptInsertion: { + 0: [32000], + 1: [-1, 32000, -2, -3], + 2: [-1, -1, 32000, -2, -2, -3, -3], }, - { - PromptInsertion: { - 0: [], - 1: [-1, -3], - 2: [-1, -1, -3, -3], - }, - PromptReplacement: { - 0: [], - 1: [-1, -3], - 2: [-1, -1, -3, -3], - }, + PromptReplacement: { + 0: [32000], + 1: [-1, 32000, -2, -3], + 2: [-1, -1, 32000, -2, -2, -3, -3], }, - ), - ( - [32000], - { - "pattern_1": PromptIndexTargets.start(), - "pattern_2": PromptIndexTargets.prefix([32000]), - "pattern_3": PromptIndexTargets.end(), + }, + ), + # Test different replacement per item + ( + [32000, 32000, 32000], + { + "pattern_1": [32000], + }, + { + "pattern_1": lambda idx: [-(idx + 1)], + }, + { + PromptInsertion: { + 0: [32000, 32000, 32000], + 1: [32000, -1, 32000, 32000], + 2: [32000, -1, -2, 32000, 32000], }, - { - "pattern_1": [-1], - "pattern_2": [-2], - "pattern_3": [-3], + PromptReplacement: { + 0: [32000, 32000, 32000], + 1: [-1, 32000, 32000], + 2: [-1, -2, 32000], }, - { - PromptInsertion: { - 0: [32000], - 1: [-1, 32000, -2, -3], - 2: [-1, -1, 32000, -2, -2, -3, -3], - }, - PromptReplacement: { - 0: [32000], - 1: [-1, 32000, -2, -3], - 2: [-1, -1, 32000, -2, -2, -3, -3], - }, + }, + ), + ( + [32000, 32000, 32000], + { + "pattern_1": PromptIndexTargets.prefix([32000]), + }, + { + "pattern_1": lambda idx: [-(idx + 1)], + }, + { + PromptInsertion: { + 0: [32000, 32000, 32000], + 1: [32000, -1, 32000, 32000], + 2: [32000, -1, -2, 32000, 32000], }, - ), - # Test different replacement per item - ( - [32000, 32000, 32000], - { - "pattern_1": [32000], + PromptReplacement: { + 0: [32000, 32000, 32000], + 1: [32000, -1, 32000, 32000], + 2: [32000, -1, -2, 32000, 32000], }, - { - "pattern_1": lambda idx: [-(idx + 1)], + }, + ), +] + + +def _placeholder(modality, item_idx, start_idx, tokens): + return PlaceholderFeaturesInfo( + modality=modality, + item_idx=item_idx, + start_idx=start_idx, + tokens=tokens, + is_embed=None, + ) + + +FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED = [ + { + PromptInsertion: { + 0: {}, + 1: { + "pattern_1": [_placeholder("pattern_1", 0, 4, [32000, 32000])], + "pattern_3": [_placeholder("pattern_3", 0, 11, [1550, 918, 1550])], }, - { - PromptInsertion: { - 0: [32000, 32000, 32000], - 1: [32000, -1, 32000, 32000], - 2: [32000, -1, -2, 32000, 32000], - }, - PromptReplacement: { - 0: [32000, 32000, 32000], - 1: [-1, 32000, 32000], - 2: [-1, -2, 32000], - }, + 2: { + "pattern_1": [ + _placeholder("pattern_1", 0, 4, [32000, 32000]), + _placeholder("pattern_1", 1, 6, [32000, 32000]), + ], + "pattern_3": [ + _placeholder("pattern_3", 0, 13, [1550, 918, 1550]), + _placeholder("pattern_3", 1, 16, [1550, 918, 1550]), + ], }, - ), - ( - [32000, 32000, 32000], - { - "pattern_1": PromptIndexTargets.prefix([32000]), + }, + PromptReplacement: { + 0: {}, + 1: { + "pattern_1": [_placeholder("pattern_1", 0, 1, [32000, 32000])], + "pattern_3": [_placeholder("pattern_3", 0, 7, [1550, 918, 1550])], }, - { - "pattern_1": lambda idx: [-(idx + 1)], + 2: {}, + }, + }, + { + PromptInsertion: {0: {}, 1: {}, 2: {}}, + PromptReplacement: {0: {}, 1: {}, 2: {}}, + }, + { + PromptInsertion: { + 0: {}, + 1: { + "pattern_1": [_placeholder("pattern_1", 0, 0, [-1])], + "pattern_2": [_placeholder("pattern_2", 0, 2, [-2])], + "pattern_3": [_placeholder("pattern_3", 0, 3, [-3])], }, - { - PromptInsertion: { - 0: [32000, 32000, 32000], - 1: [32000, -1, 32000, 32000], - 2: [32000, -1, -2, 32000, 32000], - }, - PromptReplacement: { - 0: [32000, 32000, 32000], - 1: [32000, -1, 32000, 32000], - 2: [32000, -1, -2, 32000, 32000], - }, + 2: { + "pattern_1": [ + _placeholder("pattern_1", 0, 0, [-1]), + _placeholder("pattern_1", 1, 1, [-1]), + ], + "pattern_2": [ + _placeholder("pattern_2", 0, 3, [-2]), + _placeholder("pattern_2", 1, 4, [-2]), + ], + "pattern_3": [ + _placeholder("pattern_3", 0, 5, [-3]), + _placeholder("pattern_3", 1, 6, [-3]), + ], }, - ), - ], + }, + PromptReplacement: { + 0: {}, + 1: { + "pattern_1": [_placeholder("pattern_1", 0, 0, [-1])], + "pattern_2": [_placeholder("pattern_2", 0, 2, [-2])], + "pattern_3": [_placeholder("pattern_3", 0, 3, [-3])], + }, + 2: { + "pattern_1": [ + _placeholder("pattern_1", 0, 0, [-1]), + _placeholder("pattern_1", 1, 1, [-1]), + ], + "pattern_2": [ + _placeholder("pattern_2", 0, 3, [-2]), + _placeholder("pattern_2", 1, 4, [-2]), + ], + "pattern_3": [ + _placeholder("pattern_3", 0, 5, [-3]), + _placeholder("pattern_3", 1, 6, [-3]), + ], + }, + }, + }, + { + PromptInsertion: { + 0: {}, + 1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]}, + 2: { + "pattern_1": [ + _placeholder("pattern_1", 0, 1, [-1]), + _placeholder("pattern_1", 1, 2, [-2]), + ] + }, + }, + PromptReplacement: { + 0: {}, + 1: {"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])]}, + 2: { + "pattern_1": [ + _placeholder("pattern_1", 0, 0, [-1]), + _placeholder("pattern_1", 1, 1, [-2]), + ] + }, + }, + }, + { + PromptInsertion: { + 0: {}, + 1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]}, + 2: { + "pattern_1": [ + _placeholder("pattern_1", 0, 1, [-1]), + _placeholder("pattern_1", 1, 2, [-2]), + ] + }, + }, + PromptReplacement: { + 0: {}, + 1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]}, + 2: { + "pattern_1": [ + _placeholder("pattern_1", 0, 1, [-1]), + _placeholder("pattern_1", 1, 2, [-2]), + ] + }, + }, + }, +] + + +@pytest.mark.parametrize( + ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501 + FIND_UPDATE_TOKENS_TEST_CASES, ) def test_find_update_tokens( prompt, @@ -769,6 +908,73 @@ def test_find_update_tokens( assert new_prompt == expected +@pytest.mark.parametrize( + ( + "prompt", + "target_by_key", + "repl_by_key", + "expected_by_update_type_mm_count", + "expected_placeholders_by_update_type_mm_count", + ), + [ + (*case, placeholder_expected) + for case, placeholder_expected in zip( + FIND_UPDATE_TOKENS_TEST_CASES, + FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED, + strict=True, + ) + ], +) +def test_apply_token_matches_with_placeholders( + prompt, + target_by_key, + repl_by_key, + expected_by_update_type_mm_count, + expected_placeholders_by_update_type_mm_count, +): + for update_type, expected_by_mm_count in expected_by_update_type_mm_count.items(): + for mm_count, expected in expected_by_mm_count.items(): + mm_prompt_updates = { + key: [ + [update_type(key, target, repl_by_key[key]).resolve(i)] + for i in range(mm_count) + ] + for key, target in target_by_key.items() + } + + new_prompt, result, placeholders = _apply_token_matches_with_placeholders( + prompt, + mm_prompt_updates, + tokenizer=None, + ) + + if any( + update_idx is None + for update_idxs in result.values() + for update_idx in update_idxs + ): + continue + + expected_placeholders = expected_placeholders_by_update_type_mm_count[ + update_type + ][mm_count] + + # Only displayed on error + print("update_type:", update_type) + print("mm_count:", mm_count) + print("mm_prompt_updates:", mm_prompt_updates) + print("new_prompt:", new_prompt) + print("result:", result) + print("placeholders:", placeholders) + + assert new_prompt == expected + assert { + modality: ph_list + for modality, ph_list in placeholders.items() + if ph_list + } == expected_placeholders + + @pytest.mark.parametrize( "repl_by_key", [ diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 0d9f8f14188f..5551e0d8b9d3 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -373,6 +373,40 @@ def _apply_token_matches( return token_ids, res + def _apply_token_matches_with_placeholders( + self, + token_ids: list[int], + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[ + list[int], + MultiModalPromptUpdatesApplyResult, + Mapping[str, list[PlaceholderFeaturesInfo]], + ]: + new_token_ids, match_result = self._apply_token_matches( + token_ids, + mm_prompt_updates, + ) + + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = { + modality: [] for modality in mm_prompt_updates + } + + if all( + all(update_idx is not None for update_idx in update_idxs) + for update_idxs in match_result.values() + ): + placeholders = dict( + self._find_mm_placeholders( + new_token_ids, + self._matched_updates_from_result( + mm_prompt_updates, + match_result, + ), + ) + ) + + return new_token_ids, match_result, placeholders + def _find_mm_placeholders( self, new_token_ids: list[int], diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index 2f5688853d62..3a01f1457aed 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -391,6 +391,40 @@ def _apply_token_matches( return token_ids, res + def _apply_token_matches_with_placeholders( + self, + token_ids: list[int], + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[ + list[int], + MultiModalPromptUpdatesApplyResult, + Mapping[str, list[PlaceholderFeaturesInfo]], + ]: + new_token_ids, match_result = self._apply_token_matches( + token_ids, + mm_prompt_updates, + ) + + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = { + modality: [] for modality in mm_prompt_updates + } + + if all( + all(update_idx is not None for update_idx in update_idxs) + for update_idxs in match_result.values() + ): + placeholders = dict( + self._find_mm_placeholders( + new_token_ids, + self._matched_updates_from_result( + mm_prompt_updates, + match_result, + ), + ) + ) + + return new_token_ids, match_result, placeholders + def _find_mm_placeholders( self, new_token_ids: list[int], diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index d35856e04f03..6dc5bdd35063 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -958,6 +958,64 @@ def apply_token_matches( return flatten_2d_lists(token_id_seqs), result +def _apply_token_matches_with_placeholders( + token_ids: list[int], + mm_prompt_updates: "MultiModalPromptUpdates", + tokenizer: TokenizerLike | None, +) -> tuple[ + list[int], + "MultiModalPromptUpdatesApplyResult", + Mapping[str, list[PlaceholderFeaturesInfo]], +]: + matched_updates, result = _plan_prompt_updates( + token_ids, + mm_prompt_updates, + tokenizer, + ) + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = { + modality: [] for modality in mm_prompt_updates + } + + new_token_ids = list[int]() + prev_end_idx = 0 + for matched_update in matched_updates: + update = matched_update.update + match = matched_update.match + matched_content = update.content.full + + if update.mode == UpdateMode.INSERT: + end_idx_to_insert = match.end_idx + elif update.mode == UpdateMode.REPLACE: + end_idx_to_insert = match.start_idx + else: + assert_never(update.mode) + + new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert]) + start_idx = len(new_token_ids) + + tokens = _seq2tokens(tokenizer, matched_content) + if tokens: + content_is_embed = update.content.is_embed + if content_is_embed is not None: + content_is_embed = content_is_embed(tokenizer, matched_content) + + placeholders[update.modality].append( + PlaceholderFeaturesInfo( + modality=update.modality, + item_idx=update.item_idx, + start_idx=start_idx, + tokens=tokens, + is_embed=content_is_embed, + ) + ) + new_token_ids.extend(tokens) + + prev_end_idx = match.end_idx + + new_token_ids.extend(token_ids[prev_end_idx:]) + return new_token_ids, result, placeholders + + def apply_text_matches( prompt: str, mm_prompt_updates: "MultiModalPromptUpdates", @@ -1671,6 +1729,22 @@ def _apply_token_matches( tokenizer = self.info.get_tokenizer() return apply_token_matches(prompt, mm_prompt_updates, tokenizer) + def _apply_token_matches_with_placeholders( + self, + token_ids: list[int], + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[ + list[int], + MultiModalPromptUpdatesApplyResult, + Mapping[str, list[PlaceholderFeaturesInfo]], + ]: + tokenizer = self.info.get_tokenizer() + return _apply_token_matches_with_placeholders( + token_ids, + mm_prompt_updates, + tokenizer, + ) + def _apply_text_matches( self, prompt: str, @@ -1689,6 +1763,25 @@ def _apply_text_matches_as_segmented_tokens( prompt, mm_prompt_updates, tokenizer ) + def _matched_updates_from_result( + self, + mm_prompt_updates: MultiModalPromptUpdates, + match_result: MultiModalPromptUpdatesApplyResult, + ) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]: + matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list) + for modality, update_idxs in match_result.items(): + for item_idx, update_idx in enumerate(update_idxs): + assert update_idx is not None, ( + "Failed to apply prompt replacement for " + f"mm_items[{modality!r}][{item_idx}]" + ) + + matched_updates[modality].append( + [mm_prompt_updates[modality][item_idx][update_idx]] + ) + + return dict(matched_updates) + def _apply_prompt_updates( self, token_ids: list[int], @@ -1697,11 +1790,24 @@ def _apply_prompt_updates( """Apply multi-modal prompt updates to token IDs.""" tokenizer = self.info.get_tokenizer() - new_token_ids, match_result = self._apply_token_matches( - token_ids, - mm_prompt_updates, + new_token_ids, match_result, placeholders = ( + self._apply_token_matches_with_placeholders( + token_ids, + mm_prompt_updates, + ) ) + if all( + all(update_idx is not None for update_idx in update_idxs) + for update_idxs in match_result.values() + ): + placeholders = { + modality: modality_placeholders + for modality, modality_placeholders in placeholders.items() + if modality_placeholders + } + return new_token_ids, placeholders + # If the search text does not represent a special token, # it may have different token IDs in the prompt, because # the tokens may go across the boundaries of the search text. @@ -1712,32 +1818,16 @@ def _apply_prompt_updates( # Since it is inefficient to search for all possible tokenizations # of the search text in the prompt, we instead perform string-based # updates on the decoded token IDs, then encode them back. - if not all( - all(update_idx is not None for update_idx in update_idxs) - for update_idxs in match_result.values() - ): - new_text, match_result = self._apply_text_matches( - _seq2text(tokenizer, token_ids, use_cache=False), - mm_prompt_updates, - ) - - new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False) - - matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list) - for modality, update_idxs in match_result.items(): - for item_idx, update_idx in enumerate(update_idxs): - assert update_idx is not None, ( - "Failed to apply prompt replacement for " - f"mm_items[{modality!r}][{item_idx}]" - ) + new_text, match_result = self._apply_text_matches( + _seq2text(tokenizer, token_ids, use_cache=False), + mm_prompt_updates, + ) - matched_updates[modality].append( - [mm_prompt_updates[modality][item_idx][update_idx]] - ) + new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False) placeholders = self._find_mm_placeholders( new_token_ids, - dict(matched_updates), + self._matched_updates_from_result(mm_prompt_updates, match_result), ) return new_token_ids, placeholders