From 762ebff04d756d75b23138b9afdc2e231f0641b8 Mon Sep 17 00:00:00 2001 From: yewentao256 Date: Wed, 28 Oct 2026 18:59:58 +0000 Subject: [PATCH 1/6] fix mypy multimodels Signed-off-by: yewentao256 --- tests/models/multimodal/conftest.py | 2 +- .../multimodal/generation/test_common.py | 6 +-- .../generation/test_granite_speech.py | 2 +- .../multimodal/generation/test_maverick.py | 2 +- .../generation/test_mm_prefix_lm.py | 2 +- .../generation/test_nemotron_parse_35.py | 6 ++- .../generation/test_nemotron_parse_35_mtp.py | 9 ++-- .../multimodal/generation/test_qwen2_vl.py | 2 + .../multimodal/generation/test_ultravox.py | 4 +- .../generation/test_vit_cudagraph.py | 5 +- .../multimodal/generation/test_voxtral.py | 2 +- .../generation/test_voxtral_realtime.py | 2 +- .../generation/vlm_utils/builders.py | 2 +- .../generation/vlm_utils/model_utils.py | 26 +++++---- .../generation_ppl_test/ppl_utils.py | 2 +- .../multimodal/pooling/test_dse_qwen2_vl.py | 2 +- .../pooling/test_embedding_gemma2.py | 6 +++ .../multimodal/pooling/test_intern_vit.py | 4 +- .../multimodal/pooling/test_llava_next.py | 2 +- tests/models/multimodal/pooling/test_phi3v.py | 4 +- .../processing/test_audio_in_video.py | 32 +++++++---- .../processing/test_audioflamingo3.py | 4 +- .../multimodal/processing/test_common.py | 13 +++-- .../multimodal/processing/test_gemma3.py | 5 +- .../multimodal/processing/test_gemma4.py | 7 ++- .../processing/test_gemma4_unified.py | 18 +++---- .../multimodal/processing/test_glm4_1v.py | 2 +- .../multimodal/processing/test_glm5next.py | 2 + .../multimodal/processing/test_h2ovl.py | 5 +- .../multimodal/processing/test_internvl.py | 5 +- .../multimodal/processing/test_kimi_k3.py | 4 +- .../multimodal/processing/test_lightonocr.py | 9 ++-- .../multimodal/processing/test_llama4.py | 15 ++++-- .../multimodal/processing/test_minicpmv.py | 4 ++ .../multimodal/processing/test_mistral3.py | 17 ++++-- .../multimodal/processing/test_mllama4.py | 11 ++-- .../multimodal/processing/test_molmo2.py | 8 ++- .../multimodal/processing/test_moondream3.py | 5 +- .../multimodal/processing/test_moss_audio.py | 29 +++++----- .../test_moss_transcribe_diarize.py | 7 ++- .../processing/test_nemotron_parse.py | 9 +++- .../processing/test_nemotron_parse_35.py | 12 +++-- .../multimodal/processing/test_nemotron_vl.py | 5 +- .../multimodal/processing/test_openvla.py | 7 ++- .../processing/test_qwen2_5_omni_embed.py | 2 +- .../multimodal/processing/test_qwen2_vl.py | 19 +++++-- .../multimodal/processing/test_qwen3_omni.py | 8 ++- .../multimodal/processing/test_qwen3_vl.py | 27 +++++++--- .../multimodal/processing/test_smolvlm.py | 6 ++- .../processing/test_tensor_schema.py | 15 +++--- .../processing/test_transformers_audio.py | 8 ++- .../processing/test_transformers_image.py | 18 +++++-- .../processing/test_transformers_video.py | 31 ++++++++--- .../processing/test_unlimited_ocr.py | 9 ++-- .../multimodal/processing/test_whisper.py | 4 ++ tests/models/multimodal/test_mapping.py | 5 +- tests/models/multimodal/test_mimo_v2_omni.py | 7 ++- .../multimodal/test_nano_nemotron_vl.py | 54 +++++++++---------- tests/models/multimodal/test_openpangu_vl.py | 7 ++- tools/pre_commit/mypy.py | 2 +- 60 files changed, 362 insertions(+), 187 deletions(-) diff --git a/tests/models/multimodal/conftest.py b/tests/models/multimodal/conftest.py index 54e545d6bf04..c00932738f47 100644 --- a/tests/models/multimodal/conftest.py +++ b/tests/models/multimodal/conftest.py @@ -10,4 +10,4 @@ # longer re-exports the typing aliases, so executing that module raises NameError. # TODO: Remove once the checkpoint fixes its imports. if not hasattr(F, "List"): - F.List = list + F.List = list # type: ignore[attr-defined] diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index 9ea2cc43cd6e..083eba8b5639 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -697,7 +697,7 @@ def _granite4_vision_vllm_to_hf_output(vllm_output, model): img_idx_to_prompt=lambda idx: "(./)\n", max_model_len=4096, max_num_seqs=2, - get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id], + get_stop_token_ids=lambda tok: [getattr(tok, "eos_id"), getattr(tok, "eot_id")], # noqa: B009 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output, patch_hf_runner=model_utils.minicpmv_25_patch_hf_runner, ), @@ -927,7 +927,7 @@ def _granite4_vision_vllm_to_hf_output(vllm_output, model): auto_cls=AutoModelForImageTextToText, vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output, marks=multi_gpu_marks(num_gpus=2), - **COMMON_BROADCAST_SETTINGS, # type: ignore + **COMMON_BROADCAST_SETTINGS, ), "llava_next-broadcast": VLMTestInfo( models=["llava-hf/llava-v1.6-mistral-7b-hf"], @@ -936,7 +936,7 @@ def _granite4_vision_vllm_to_hf_output(vllm_output, model): auto_cls=AutoModelForImageTextToText, vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output, marks=multi_gpu_marks(num_gpus=2), - **COMMON_BROADCAST_SETTINGS, # type: ignore + **COMMON_BROADCAST_SETTINGS, ), ### Custom input edge-cases for specific models "intern_vl-diff-patches": VLMTestInfo( diff --git a/tests/models/multimodal/generation/test_granite_speech.py b/tests/models/multimodal/generation/test_granite_speech.py index e1fee635ff66..f5ee95302543 100644 --- a/tests/models/multimodal/generation/test_granite_speech.py +++ b/tests/models/multimodal/generation/test_granite_speech.py @@ -169,7 +169,7 @@ def test_models( hf_runner, vllm_runner, [ - ([HF_AUDIO_PROMPT], [audio]), + ([HF_AUDIO_PROMPT], [(audio, int(sr))]), ], model, dtype=dtype, diff --git a/tests/models/multimodal/generation/test_maverick.py b/tests/models/multimodal/generation/test_maverick.py index 0f6bec107fc2..a44491dfec31 100644 --- a/tests/models/multimodal/generation/test_maverick.py +++ b/tests/models/multimodal/generation/test_maverick.py @@ -523,7 +523,7 @@ def save_weights_to_safetensors( print(f"Saved shard {i + 1}/{len(shards)}: {filename}") # Create index file - index_data = { + index_data: dict[str, Any] = { "metadata": { "total_size": sum( tensor.numel() * tensor.element_size() for tensor in weights.values() diff --git a/tests/models/multimodal/generation/test_mm_prefix_lm.py b/tests/models/multimodal/generation/test_mm_prefix_lm.py index 10ff63f6c0fe..e1e3e1223266 100644 --- a/tests/models/multimodal/generation/test_mm_prefix_lm.py +++ b/tests/models/multimodal/generation/test_mm_prefix_lm.py @@ -50,7 +50,7 @@ def _get_hf_prefill_hidden(hf_model: HfRunner, image: Any): inputs = hf_model.get_inputs([PROMPT], images=[image])[0] with torch.no_grad(): outputs = hf_model.model.model( - **hf_model.wrap_device(inputs), + **hf_model.wrap_device(dict(inputs)), use_cache=False, ) return outputs.last_hidden_state[0].detach().float().cpu() diff --git a/tests/models/multimodal/generation/test_nemotron_parse_35.py b/tests/models/multimodal/generation/test_nemotron_parse_35.py index bb8fbf51d937..e151e184f478 100644 --- a/tests/models/multimodal/generation/test_nemotron_parse_35.py +++ b/tests/models/multimodal/generation/test_nemotron_parse_35.py @@ -35,7 +35,7 @@ def generate_hf_logprobs_from_logits( outputs = [] for model_inputs in inputs: generated = hf_model.model.generate( - **hf_model.wrap_device(model_inputs), + **hf_model.wrap_device(dict(model_inputs)), do_sample=False, max_new_tokens=max_tokens, output_logits=True, @@ -51,7 +51,9 @@ def generate_hf_logprobs_from_logits( } ) token_ids = generated.sequences[0, -len(logprobs) :].tolist() - outputs.append((token_ids, hf_model.tokenizer.decode(token_ids), logprobs)) + decoded = hf_model.tokenizer.decode(token_ids) + assert isinstance(decoded, str) + outputs.append((token_ids, decoded, logprobs)) return outputs diff --git a/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py b/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py index 918bc51ea657..55d5108ee09b 100644 --- a/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py +++ b/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py @@ -23,10 +23,11 @@ def test_parse_mtp_config_routes_only_causal_prefix_architecture() -> None: - config = PreTrainedConfig( - architectures=["RadioMBartPrefixForCausalLM"], - decoder=PreTrainedConfig(causal_vision_prefix=True, vocab_size=8, d_model=4), - ) + config = PreTrainedConfig(architectures=["RadioMBartPrefixForCausalLM"]) + config.decoder = PreTrainedConfig() + config.decoder.causal_vision_prefix = True + config.decoder.vocab_size = 8 + config.decoder.d_model = 4 draft = SpeculativeConfig.hf_config_override(config) diff --git a/tests/models/multimodal/generation/test_qwen2_vl.py b/tests/models/multimodal/generation/test_qwen2_vl.py index 4508135b2bb9..ea74d117b302 100644 --- a/tests/models/multimodal/generation/test_qwen2_vl.py +++ b/tests/models/multimodal/generation/test_qwen2_vl.py @@ -126,6 +126,7 @@ def batch_make_image_embeddings( image_processor = processor.image_processor mm_config = llm.get_llm().model_config.multimodal_config + assert mm_config is not None mm_kwargs = mm_config.merge_mm_processor_kwargs({}) preprocess_result = image_processor.preprocess( images=images, return_tensors="pt", **mm_kwargs @@ -211,6 +212,7 @@ def batch_make_video_embeddings( video_processor = processor.video_processor mm_config = llm.get_llm().model_config.multimodal_config + assert mm_config is not None mm_kwargs = mm_config.merge_mm_processor_kwargs({}) preprocess_result = video_processor.preprocess( videos=videos, return_tensors="pt", **mm_kwargs diff --git a/tests/models/multimodal/generation/test_ultravox.py b/tests/models/multimodal/generation/test_ultravox.py index 72f473835800..8ae5e573b4fb 100644 --- a/tests/models/multimodal/generation/test_ultravox.py +++ b/tests/models/multimodal/generation/test_ultravox.py @@ -24,7 +24,7 @@ MULTI_AUDIO_PROMPT = "Describe each of the audios above." -AudioTuple = tuple[np.ndarray, int] +AudioTuple = tuple[np.ndarray, float] VLLM_PLACEHOLDER = "<|audio|>" HF_PLACEHOLDER = "<|audio|>" @@ -92,7 +92,7 @@ def _get_prompt(audio_count, question, placeholder): def run_multi_audio_test( vllm_runner: type[VllmRunner], - prompts_and_audios: list[tuple[str, list[AudioTuple]]], + prompts_and_audios: list[tuple[str | list[int], list[AudioTuple]]], model: str, *, dtype: str, diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index e80dd9dc8515..6db55fa29d83 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -34,7 +34,7 @@ class VitCudagraphTestConfig: def params_with_marks( configs: dict[str, VitCudagraphTestConfig], -) -> list[pytest.param]: +) -> list[object]: return [ pytest.param(model_id, marks=cfg.marks) for model_id, cfg in configs.items() ] @@ -520,6 +520,7 @@ def test_vit_cudagraph_video(model_id, vllm_runner, video_assets): "baby_reading": config.video_prompt, # type: ignore[typeddict-item] } ) + sampled_vids: list if config.needs_video_metadata: sampled_vids = [ sample_frames_with_video_metadata( @@ -576,7 +577,7 @@ def _check_eonly_encoder_outputs(worker, batches): for batch in batches: kwargs = {key: value.to(runner.device) for key, value in batch.items()} actual = manager.execute(kwargs) - expected = runner.model.embed_multimodal(**kwargs) + expected = getattr(runner.model, "embed_multimodal")(**kwargs) # noqa: B009 assert len(actual) == len(expected) for output, reference in zip(actual, expected): assert torch.isfinite(output).all() diff --git a/tests/models/multimodal/generation/test_voxtral.py b/tests/models/multimodal/generation/test_voxtral.py index 3c9d2ace8cca..434df130a080 100644 --- a/tests/models/multimodal/generation/test_voxtral.py +++ b/tests/models/multimodal/generation/test_voxtral.py @@ -58,7 +58,7 @@ def test_models_with_multiple_audios( vllm_prompt = _get_prompt(audio_assets, MULTI_AUDIO_PROMPT) run_multi_audio_test( vllm_runner, - [(vllm_prompt, [a.audio_and_sample_rate for a in audio_assets])], # type: ignore[list-item] + [(vllm_prompt, [a.audio_and_sample_rate for a in audio_assets])], MODEL_NAME, dtype=dtype, max_tokens=max_tokens, diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index e05930b0f36b..719ff4183b12 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -138,7 +138,7 @@ async def async_engine(request): def _build_inputs( audio_assets: list[AudioAsset], tokenizer: MistralTokenizer ) -> tuple[list[dict], list[SamplingParams]]: - audio_config = tokenizer.instruct_tokenizer.tokenizer.audio + audio_config = getattr(tokenizer.instruct_tokenizer.tokenizer, "audio") # noqa: B009 inputs = [] sampling_params = [] for audio_asset in audio_assets: diff --git a/tests/models/multimodal/generation/vlm_utils/builders.py b/tests/models/multimodal/generation/vlm_utils/builders.py index 1b7e2347be2f..e73c69c13897 100644 --- a/tests/models/multimodal/generation/vlm_utils/builders.py +++ b/tests/models/multimodal/generation/vlm_utils/builders.py @@ -224,7 +224,7 @@ def build_embedding_inputs_from_test_info( images = [asset.pil_image for asset in image_assets] embeds = test_info.convert_assets_to_embeddings(image_assets) if test_info.dtype != "auto": - dtype = getattr(torch, test_info.dtype) # type: ignore + dtype = getattr(torch, test_info.dtype) embeds = [e.to(dtype=dtype) for e in embeds] assert len(images) == len(model_prompts) diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 9acbe2c0d6b3..4b7949d07c5a 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -22,6 +22,7 @@ GenerationMixin, ) from transformers.masking_utils import create_causal_mask +from transformers.utils import ModelOutput from transformers.video_utils import VideoMetadata from vllm.logprobs import SampleLogprobs @@ -105,7 +106,9 @@ def _llava_vllm_to_hf_output( hf_output_str = output_str[1:] if output_str[0] == " " else output_str if hf_output_ids[-1] == eos_token_id: - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) + decoded_eos = tokenizer.decode(eos_token_id) + assert isinstance(decoded_eos, str) + hf_output_str = hf_output_str + decoded_eos return hf_output_ids, hf_output_str, out_logprobs @@ -137,7 +140,9 @@ def llava_onevision_vllm_to_hf_output( hf_output_str = output_str if hf_output_ids[-1] == eos_token_id: - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) + decoded_eos = tokenizer.decode(eos_token_id) + assert isinstance(decoded_eos, str) + hf_output_str = hf_output_str + decoded_eos return hf_output_ids, hf_output_str, out_logprobs @@ -179,7 +184,9 @@ def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> Runner hf_output_str = output_str if hf_output_ids[-1] == eos_token_id: - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) + decoded_eos = tokenizer.decode(eos_token_id) + assert isinstance(decoded_eos, str) + hf_output_str = hf_output_str + decoded_eos return hf_output_ids, hf_output_str, out_logprobs @@ -286,7 +293,9 @@ def gemma3_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOut hf_output_str = output_str if hf_output_ids[-1] == eos_token_id: - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) + decoded_eos = tokenizer.decode(eos_token_id) + assert isinstance(decoded_eos, str) + hf_output_str = hf_output_str + decoded_eos return hf_output_ids, hf_output_str, out_logprobs @@ -572,11 +581,10 @@ def patched_forward( # Prepare attention mask attention_mask = create_causal_mask( config=self.config, - input_embeds=inputs_embeds, + inputs_embeds=inputs_embeds, attention_mask=attention_mask, past_key_values=past_key_values, position_ids=position_ids, - cache_position=cache_position, ) # Initialize and collect hidden states @@ -800,7 +808,7 @@ def _internvl_generate( input_ids: torch.FloatTensor, attention_mask: torch.LongTensor | None = None, **generate_kwargs, -) -> torch.LongTensor: +) -> torch.Tensor | ModelOutput: """Generate method for InternVL2 model without fixed use_cache.""" assert self.img_context_token_id is not None target_dtype = next(self.parameters()).dtype @@ -828,7 +836,7 @@ def _internvl_generate( if not isinstance(self.language_model, GenerationMixin): pytest.skip("HF impl is not compatible with current transformers") - outputs = self.language_model.generate( + outputs = getattr(self.language_model, "generate")( # noqa: B009 **forward_kwargs, **generate_kwargs, ) @@ -1207,7 +1215,7 @@ def patched_generate(*args, **kwargs): return output hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign] - hf_model.model.generate = patched_generate # type: ignore[method-assign] + hf_model.model.generate = patched_generate return hf_model diff --git a/tests/models/multimodal/generation_ppl_test/ppl_utils.py b/tests/models/multimodal/generation_ppl_test/ppl_utils.py index c612eb7141da..d19764740995 100644 --- a/tests/models/multimodal/generation_ppl_test/ppl_utils.py +++ b/tests/models/multimodal/generation_ppl_test/ppl_utils.py @@ -17,7 +17,7 @@ PPL_TOL = 0.01 -@torch.inference_mode +@torch.inference_mode() def vqa_ppl_test( hf_runner, vllm_runner, diff --git a/tests/models/multimodal/pooling/test_dse_qwen2_vl.py b/tests/models/multimodal/pooling/test_dse_qwen2_vl.py index 0fff8f40ac74..8b975c0ada48 100644 --- a/tests/models/multimodal/pooling/test_dse_qwen2_vl.py +++ b/tests/models/multimodal/pooling/test_dse_qwen2_vl.py @@ -180,7 +180,7 @@ def test_models_text( hf_runner, vllm_runner, input_texts, - input_images, # type: ignore + input_images, embed_texts, model, dtype=dtype, diff --git a/tests/models/multimodal/pooling/test_embedding_gemma2.py b/tests/models/multimodal/pooling/test_embedding_gemma2.py index 0dcb72406da5..ff4f7e9fb167 100644 --- a/tests/models/multimodal/pooling/test_embedding_gemma2.py +++ b/tests/models/multimodal/pooling/test_embedding_gemma2.py @@ -9,6 +9,8 @@ import pytest import torch +from vllm.multimodal.video import EmbeddingGemma2VideoBackend + MODEL = os.environ.get("EG2_MODEL_PATH") needs_model = pytest.mark.skipif(not MODEL, reason="EG2_MODEL_PATH not set") TEXT_COS, MM_COS = 0.999, 0.999 @@ -348,6 +350,7 @@ def test_video_loader_real_video(vllm_runner, st_model): video_path = "/projects/gemma4-vllm/videos/test_intel.mp4" backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"] + assert issubclass(backend_cls, EmbeddingGemma2VideoBackend) with open(video_path, "rb") as f: data = f.read() @@ -396,6 +399,7 @@ def test_video_loader_duplicate_expansion_e2e(vllm_runner, st_model): from vllm.multimodal.video import VIDEO_LOADER_REGISTRY backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"] + assert issubclass(backend_cls, EmbeddingGemma2VideoBackend) # 4s clip at 0.5 fps = 2 unique frames lowfps_path = "/tmp/test_lowfps_05.mp4" _ensure_synthetic_video(lowfps_path, duration=4.0, fps=0.5, width=64, height=48) @@ -436,6 +440,7 @@ def test_video_loader_indices(): ) backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"] + assert issubclass(backend_cls, EmbeddingGemma2VideoBackend) hf_proc = EmbeddingGemma2VideoProcessor() # Case 1: clip shorter than 32s (e.g. 5s @ 24fps = 120 frames) @@ -521,6 +526,7 @@ def test_video_loader_duplicate_expansion(): ) backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"] + assert issubclass(backend_cls, EmbeddingGemma2VideoBackend) hf_proc = EmbeddingGemma2VideoProcessor() # 4s clip @ 0.5fps = 2 frames total. Requested target fps = 1.0 -> 4 frames. diff --git a/tests/models/multimodal/pooling/test_intern_vit.py b/tests/models/multimodal/pooling/test_intern_vit.py index 4d5e8728589c..7b5db69c6b9a 100644 --- a/tests/models/multimodal/pooling/test_intern_vit.py +++ b/tests/models/multimodal/pooling/test_intern_vit.py @@ -49,14 +49,14 @@ def _patched_init(self, *args, **kwargs): _orig_init(self, *args, **kwargs) self.all_tied_weights_keys = {} - PreTrainedModel.__init__ = _patched_init + PreTrainedModel.__init__ = _patched_init # type: ignore[method-assign] try: hf_model = AutoModel.from_pretrained( model, dtype=torch_dtype, trust_remote_code=True ).to(DEVICE_TYPE) finally: - PreTrainedModel.__init__ = _orig_init + PreTrainedModel.__init__ = _orig_init # type: ignore[method-assign] hf_outputs_per_image = [ hf_model(pixel_value.to(DEVICE_TYPE)).last_hidden_state for pixel_value in pixel_values diff --git a/tests/models/multimodal/pooling/test_llava_next.py b/tests/models/multimodal/pooling/test_llava_next.py index 97bdaea4ecbe..a329ce4fece3 100644 --- a/tests/models/multimodal/pooling/test_llava_next.py +++ b/tests/models/multimodal/pooling/test_llava_next.py @@ -126,7 +126,7 @@ def test_models_text( hf_runner, vllm_runner, input_texts, - input_images, # type: ignore + input_images, model, dtype=dtype, ) diff --git a/tests/models/multimodal/pooling/test_phi3v.py b/tests/models/multimodal/pooling/test_phi3v.py index 35df42127eca..7546b04f06c2 100644 --- a/tests/models/multimodal/pooling/test_phi3v.py +++ b/tests/models/multimodal/pooling/test_phi3v.py @@ -129,7 +129,7 @@ def test_models_text( hf_runner, vllm_runner, input_texts, - input_images, # type: ignore + input_images, model, dtype=dtype, ) @@ -192,7 +192,7 @@ def test_models_image_special_tokens_processing( return_tensors="pt", ) - image_token_id = hf_processor.get_special_image_token_id() + image_token_id = getattr(hf_processor, "get_special_image_token_id")() # noqa: B009 hf_prompt_token_ids = [ image_token_id if token_id < 0 else token_id for token_id in hf_inputs["input_ids"][0].tolist() diff --git a/tests/models/multimodal/processing/test_audio_in_video.py b/tests/models/multimodal/processing/test_audio_in_video.py index 6b77e8834e12..a8e164d7f39d 100644 --- a/tests/models/multimodal/processing/test_audio_in_video.py +++ b/tests/models/multimodal/processing/test_audio_in_video.py @@ -178,9 +178,11 @@ def test_audio_in_video_metadata_only( mm_config.enable_mm_embeds = True mm_config.allow_missing_mm_embeddings = True processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - prompt = "".join( + video_placeholders = [ model_cls.get_placeholder_str("video", i) for i in range(num_videos) - ) + ] + assert all(isinstance(token, str) for token in video_placeholders) + prompt = "".join(token for token in video_placeholders if token is not None) kwargs = {"use_audio_in_video": True} uuids = { modality: [f"source-{i}" for i in range(num_videos)] @@ -192,7 +194,9 @@ def test_audio_in_video_metadata_only( mm_data["image"] = [Image.new("RGB", (64, 64), "green")] uuids["image"] = ["image-source"] - prompt = model_cls.get_placeholder_str("image", 0) + prompt + image_placeholder = model_cls.get_placeholder_str("image", 0) + assert image_placeholder is not None + prompt = image_placeholder + prompt raw = processor( prompt, mm_items=processor.info.parse_mm_data(mm_data), @@ -237,9 +241,16 @@ def test_audio_in_video_metadata_only( "uuid": "image-source", }, ) - parse = parse_chat_messages_async if is_async else parse_chat_messages - parsed = parse(messages, ctx.model_config, "openai", mm_processor_kwargs=kwargs) - conversation, metadata, remote_uuids = asyncio.run(parsed) if is_async else parsed + if is_async: + conversation, metadata, remote_uuids = asyncio.run( + parse_chat_messages_async( + messages, ctx.model_config, "openai", mm_processor_kwargs=kwargs + ) + ) + else: + conversation, metadata, remote_uuids = parse_chat_messages( + messages, ctx.model_config, "openai", mm_processor_kwargs=kwargs + ) assert conversation[0]["content"] == ( ([{"type": "image"}] if has_image else []) + [{"type": "video"}] * num_videos ) @@ -271,7 +282,7 @@ def unexpected_preprocessing(*args, **kwargs): ) model = model_cls.__new__(model_cls) - model.config = processor.info.get_hf_config() + setattr(model, "config", processor.info.get_hf_config()) # noqa: B010 remote_features = [ MultiModalFeatureSpec( data=remote["mm_kwargs"][f.modality][int(f.identifier.split("-")[-1])], @@ -281,10 +292,9 @@ def unexpected_preprocessing(*args, **kwargs): ) for f in features ] - raw_positions, raw_delta = model.get_mrope_input_positions( - raw["prompt_token_ids"], features - ) - remote_positions, remote_delta = model.get_mrope_input_positions( + get_positions = getattr(model, "get_mrope_input_positions") # noqa: B009 + raw_positions, raw_delta = get_positions(raw["prompt_token_ids"], features) + remote_positions, remote_delta = get_positions( remote["prompt_token_ids"], remote_features ) assert raw_delta == remote_delta diff --git a/tests/models/multimodal/processing/test_audioflamingo3.py b/tests/models/multimodal/processing/test_audioflamingo3.py index c3adccb27d59..16182eac3052 100644 --- a/tests/models/multimodal/processing/test_audioflamingo3.py +++ b/tests/models/multimodal/processing/test_audioflamingo3.py @@ -134,7 +134,9 @@ def test_dummy_data_generation(mock_ctx): assert len(dummy_data["audio"]) == 2 expected_len = 60 * 16000 - assert len(dummy_data["audio"][0]) == expected_len + audio_data = dummy_data["audio"] + assert isinstance(audio_data, list) + assert len(audio_data[0]) == expected_len def test_audio_token_count_matches_hf_processor_math(): diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 667907c3e55c..f830843d0dda 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -43,6 +43,7 @@ def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict: """Add metadata to video mm_data.""" + assert isinstance(mm_data, dict) def create_metadata(frames: np.ndarray): num_frames = len(frames) @@ -71,6 +72,7 @@ def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict: """Patch the multimodal data for GLM-ASR model. GLM-ASR requires text and audio to match 1:1, so we limit audio to 1. """ + assert isinstance(mm_data, dict) if "audio" in mm_data: audio = mm_data["audio"] if isinstance(audio, list) and len(audio) > 1: @@ -218,7 +220,7 @@ def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch): mm_config.enable_mm_embeds = True mm_config.allow_missing_mm_embeddings = True processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - sample_rate = processor.info.get_feature_extractor().sampling_rate + sample_rate = getattr(processor.info, "get_feature_extractor")().sampling_rate # noqa: B009 audios = [ np.zeros(int(duration * sample_rate), dtype=np.float32) for duration in durations @@ -333,6 +335,7 @@ def _test_processing_correctness( ) # Ensure that the cache can fit all of the data # (set after because ModelConfig would set it to 0 for encoder-decoder models) + assert model_config.multimodal_config is not None model_config.multimodal_config.mm_processor_cache_gb = 2048 model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config) @@ -353,11 +356,13 @@ def _test_processing_correctness( def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions: if modality == "video": - return VideoDummyOptions(count=count) + return VideoDummyOptions( + count=count, num_frames=None, width=None, height=None + ) if modality == "image": - return ImageDummyOptions(count=count) + return ImageDummyOptions(count=count, width=None, height=None) if modality == "audio": - return AudioDummyOptions(count=count) + return AudioDummyOptions(count=count, length=None) return BaseDummyOptions(count=count) # Assign normalized DummyOptions to the model config diff --git a/tests/models/multimodal/processing/test_gemma3.py b/tests/models/multimodal/processing/test_gemma3.py index 2b4c213695ee..114799b068f5 100644 --- a/tests/models/multimodal/processing/test_gemma3.py +++ b/tests/models/multimodal/processing/test_gemma3.py @@ -4,6 +4,7 @@ import pytest import torch +from vllm.model_executor.models.gemma3_mm import Gemma3ProcessingInfo from vllm.model_executor.models.gemma3n_audio_utils import ( adjust_audio_features_to_expected_length, ) @@ -162,6 +163,7 @@ def test_get_image_size_with_most_features( limit_mm_per_prompt={"image": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Gemma3ProcessingInfo) hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs) @@ -174,7 +176,7 @@ def test_get_image_size_with_most_features( ) prompt = "" - image_seq_length = hf_processor.image_seq_length + image_seq_length = getattr(hf_processor, "image_seq_length") # noqa: B009 for asset in image_assets: mm_data = {"image": [asset.pil_image]} @@ -185,5 +187,6 @@ def test_get_image_size_with_most_features( ) mm_kwargs_data = processed_inputs["mm_kwargs"].get_data() num_patches_tensor = mm_kwargs_data["num_patches"] + assert isinstance(num_patches_tensor, torch.Tensor) tokens = int(num_patches_tensor.item()) * image_seq_length assert tokens <= max_tokens diff --git a/tests/models/multimodal/processing/test_gemma4.py b/tests/models/multimodal/processing/test_gemma4.py index f9d935991e90..c2ba4a75b342 100644 --- a/tests/models/multimodal/processing/test_gemma4.py +++ b/tests/models/multimodal/processing/test_gemma4.py @@ -14,6 +14,7 @@ from vllm.model_executor.models.gemma4_mm import ( Gemma4ForConditionalGeneration, Gemma4ImagePixelInputs, + Gemma4ProcessingInfo, ) from vllm.model_executor.models.interfaces import supports_multimodal from vllm.multimodal import MULTIMODAL_REGISTRY @@ -98,6 +99,7 @@ def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens( limit_mm_per_prompt={"image": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Gemma4ProcessingInfo) num_soft_tokens = processor.info._compute_num_soft_tokens( image_width=image_width, @@ -190,6 +192,7 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str): limit_mm_per_prompt={"image": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Gemma4ProcessingInfo) image = PILImage.new("RGB", (1000, 1000), color="white") image_size = image.size mm_items = processor.info.parse_mm_data({"image": image}) @@ -278,7 +281,7 @@ def test_video_cache_is_independent_of_image_kwargs( def process(mm_kwargs, cache): return processor( - hf_processor.image_token + hf_processor.video_token, + getattr(hf_processor, "image_token") + getattr(hf_processor, "video_token"), # noqa: B009 mm_items, mm_uuid_items={"video": [video_uuid]}, hf_processor_mm_kwargs=mm_kwargs, @@ -397,7 +400,7 @@ def test_encoder_chunk_no_free_memory_falls_back_to_one(): def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration: - model = object.__new__(Gemma4ForConditionalGeneration) + model = object.__new__(Gemma4ForConditionalGeneration) # type: ignore[type-abstract] vision_config = SimpleNamespace( patch_size=16, pooling_kernel_size=3, default_output_length=280 ) diff --git a/tests/models/multimodal/processing/test_gemma4_unified.py b/tests/models/multimodal/processing/test_gemma4_unified.py index 28c1018c6602..bd5a15c91fb2 100644 --- a/tests/models/multimodal/processing/test_gemma4_unified.py +++ b/tests/models/multimodal/processing/test_gemma4_unified.py @@ -15,6 +15,7 @@ ) from vllm.model_executor.models.gemma4_unified import ( Gemma4UnifiedForConditionalGeneration, + Gemma4UnifiedProcessingInfo, Gemma4UnifiedVisionEmbedder, ) from vllm.multimodal import MULTIMODAL_REGISTRY @@ -81,6 +82,7 @@ def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens( limit_mm_per_prompt={"image": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Gemma4UnifiedProcessingInfo) num_soft_tokens = processor.info._compute_num_soft_tokens( image_width=image_width, @@ -171,6 +173,7 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str): limit_mm_per_prompt={"image": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Gemma4UnifiedProcessingInfo) image = PILImage.new("RGB", (1000, 1000), color="white") image_size = image.size mm_items = processor.info.parse_mm_data({"image": image}) @@ -315,7 +318,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return self.embedding_projection(x) -class _MockUnifiedModel: +class _MockUnifiedModel(Gemma4UnifiedForConditionalGeneration): + # Initialize only the modules needed by the borrowed vision methods. def __init__( self, embed_dim: int = 48, @@ -323,6 +327,7 @@ def __init__( dtype: torch.dtype = torch.float32, device: str = "cpu", ): + torch.nn.Module.__init__(self) self.vision_embedder = _build_test_embedder( embed_dim=embed_dim, max_pos=max_pos, dtype=dtype, device=device ) @@ -330,11 +335,6 @@ def __init__( embed_dim=embed_dim, dtype=dtype, device=device ) - _vision_chunk = staticmethod(Gemma4UnifiedForConditionalGeneration._vision_chunk) - _embed_vision_patches = Gemma4UnifiedForConditionalGeneration._embed_vision_patches - _process_image_input = Gemma4UnifiedForConditionalGeneration._process_image_input - _process_video_input = Gemma4UnifiedForConditionalGeneration._process_video_input - @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) @pytest.mark.parametrize("shape", [(16, 2), (2, 8, 2), (1, 1, 2)]) @@ -407,7 +407,7 @@ def test_process_image_input_batching_and_chunking(): """ from unittest.mock import patch - model = _MockUnifiedModel() + model = _MockUnifiedModel() # type: ignore[abstract] # 1. 3D stacked image tensor pv = torch.randn(2, 5, 48) @@ -446,7 +446,7 @@ def test_process_image_input_batching_and_chunking(): def test_process_video_input_batching(): """Verify _process_video_input with frame counts and lists of tensors.""" - model = _MockUnifiedModel() + model = _MockUnifiedModel() # type: ignore[abstract] # 2 videos: 3 frames and 2 frames, each frame has 4 patches pv_vid = torch.randn(5, 4, 48) @@ -487,7 +487,7 @@ def test_process_video_input_batching(): def test_cuda_execution_with_gpu_sync_allowed(): """Verify vision processing on CUDA under gpu_sync_allowed.""" device = "cuda:0" - model = _MockUnifiedModel(device=device, dtype=torch.bfloat16) + model = _MockUnifiedModel(device=device, dtype=torch.bfloat16) # type: ignore[abstract] pv = torch.randn(2, 5, 48, device=device, dtype=torch.bfloat16) pp = torch.randint(0, 100, (2, 5, 2), device=device) diff --git a/tests/models/multimodal/processing/test_glm4_1v.py b/tests/models/multimodal/processing/test_glm4_1v.py index adf7219db1b7..18cd4c634aad 100644 --- a/tests/models/multimodal/processing/test_glm4_1v.py +++ b/tests/models/multimodal/processing/test_glm4_1v.py @@ -136,7 +136,7 @@ def test_processor_override( # Ensure we have the right number of placeholders per num_crops size hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token_id = hf_processor.image_token_id + image_token_id = getattr(hf_processor, "image_token_id") # noqa: B009 video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0] diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py index 111fb982cf7d..e9f3a48ec425 100644 --- a/tests/models/multimodal/processing/test_glm5next.py +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -177,10 +177,12 @@ def test_mm_device_do_normalize(): normalized_inputs = processor(prompt, mm_items=mm_items) normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] + assert isinstance(normalized_values, torch.Tensor) ctx.model_config.multimodal_config.mm_device_do_normalize = True raw_inputs = processor(prompt, mm_items=mm_items) raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] + assert isinstance(raw_values, torch.Tensor) assert raw_values.dtype == torch.uint8 input_norm = build_mm_input_norm(ctx.model_config).to(device) diff --git a/tests/models/multimodal/processing/test_h2ovl.py b/tests/models/multimodal/processing/test_h2ovl.py index 5a54484e1bdd..bc43130d04f1 100644 --- a/tests/models/multimodal/processing/test_h2ovl.py +++ b/tests/models/multimodal/processing/test_h2ovl.py @@ -120,7 +120,9 @@ def _run_check( # Ensure we have the right number of placeholders per num_crops size image_token_id = tokenizer.convert_tokens_to_ids("") img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) - pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape + pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"] + assert isinstance(pixel_values, torch.Tensor) + pixel_shape = pixel_values.shape assert img_tok_count == 256 * total_expected_num_patches assert pixel_shape[0] == total_expected_num_patches @@ -204,6 +206,7 @@ def test_image_cache_identity_across_image_counts( ) processor = MULTIMODAL_REGISTRY.create_processor(config) tokenizer = processor.info.get_tokenizer() + images: list[torch.Tensor] | list[Image.Image] if use_embeds: images = [torch.ones(8, config.get_hidden_size())] * 3 else: diff --git a/tests/models/multimodal/processing/test_internvl.py b/tests/models/multimodal/processing/test_internvl.py index ab5fbe9a0130..de126c2c95fb 100644 --- a/tests/models/multimodal/processing/test_internvl.py +++ b/tests/models/multimodal/processing/test_internvl.py @@ -5,6 +5,7 @@ from collections.abc import Mapping import pytest +import torch from PIL import Image from transformers import PreTrainedConfig @@ -75,7 +76,9 @@ def _run_check( # Ensure we have the right number of placeholders per num_crops size image_token_id = tokenizer.convert_tokens_to_ids("") img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) - pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape + pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"] + assert isinstance(pixel_values, torch.Tensor) + pixel_shape = pixel_values.shape assert img_tok_count == 256 * total_expected_num_patches assert pixel_shape[0] == total_expected_num_patches diff --git a/tests/models/multimodal/processing/test_kimi_k3.py b/tests/models/multimodal/processing/test_kimi_k3.py index 62c99b24f143..599b0413e0bc 100644 --- a/tests/models/multimodal/processing/test_kimi_k3.py +++ b/tests/models/multimodal/processing/test_kimi_k3.py @@ -23,7 +23,7 @@ def image_processors(): ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1}) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - k25_fused = processor.info.image_processor + k25_fused = getattr(processor.info, "image_processor") # noqa: B009 assert isinstance(k25_fused, KimiK25FusedVisionProcessor) k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True) return k25_fused, k3_hf @@ -68,10 +68,12 @@ def test_mm_device_do_normalize(): normalized_inputs = processor(prompt, mm_items=mm_items) normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] + assert isinstance(normalized_values, torch.Tensor) ctx.model_config.multimodal_config.mm_device_do_normalize = True raw_inputs = processor(prompt, mm_items=mm_items) raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] + assert isinstance(raw_values, torch.Tensor) assert raw_values.dtype == torch.uint8 input_norm = build_mm_input_norm(ctx.model_config).to(device) diff --git a/tests/models/multimodal/processing/test_lightonocr.py b/tests/models/multimodal/processing/test_lightonocr.py index 3966bf8cb4dd..34479ed47671 100644 --- a/tests/models/multimodal/processing/test_lightonocr.py +++ b/tests/models/multimodal/processing/test_lightonocr.py @@ -35,16 +35,17 @@ def test_lightonocr_image_embeddings_preserve_lengths_and_values( for i, size in enumerate(feature_sizes) ] image_input = torch.stack(embeddings) if tensor_input else embeddings - prompt = " ".join([hf_processor.image_token] * len(embeddings)) + prompt = " ".join([getattr(hf_processor, "image_token")] * len(embeddings)) # noqa: B009 if prompt_as_tokens: prompt = ctx.get_tokenizer().encode(prompt) items = processor.info.parse_mm_data({"image": image_input}) processed = processor(prompt, mm_items=items) - assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum( - feature_sizes - ) + assert processed["prompt_token_ids"].count( + getattr(hf_processor, "image_token_id") # noqa: B009 + ) == sum(feature_sizes) assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes for item, expected in zip(processed["mm_kwargs"]["image"], embeddings): + assert item is not None torch.testing.assert_close(item.get_data()["image_embeds"], expected) diff --git a/tests/models/multimodal/processing/test_llama4.py b/tests/models/multimodal/processing/test_llama4.py index 4bc2e5909980..e4743396c869 100644 --- a/tests/models/multimodal/processing/test_llama4.py +++ b/tests/models/multimodal/processing/test_llama4.py @@ -3,7 +3,9 @@ """Tests for Llama4's multimodal preprocessing kwargs.""" import pytest +import torch +from vllm.model_executor.models.mllama4 import Mllama4ProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from ....conftest import ImageTestAssets @@ -31,6 +33,7 @@ def test_processor_override( mm_processor_cache_gb=mm_processor_cache_gb, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Mllama4ProcessingInfo) config = processor.info.get_hf_config() tokenizer = processor.info.get_tokenizer() hf_processor = processor.info.get_hf_processor() @@ -60,16 +63,22 @@ def test_processor_override( prompt_token_ids = processed_inputs["prompt_token_ids"] assert prompt_token_ids.count(config.boi_token_index) == num_imgs assert prompt_token_ids.count(config.eoi_token_index) == num_imgs - assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs + assert ( + prompt_token_ids.count(vocab[getattr(hf_processor, "image_token")]) == num_imgs # noqa: B009 + ) aspect_ratios = mm_data["aspect_ratios"] + assert isinstance(aspect_ratios, torch.Tensor) num_x_separators = num_y_separators = 0 for tiles_y, tiles_x in aspect_ratios: if tiles_x * tiles_y > 1: num_x_separators += (tiles_x - 1) * tiles_y num_y_separators += tiles_y - assert prompt_token_ids.count(vocab[hf_processor.tile_token]) == num_x_separators assert ( - prompt_token_ids.count(vocab[hf_processor.tile_global_token]) + prompt_token_ids.count(vocab[getattr(hf_processor, "tile_token")]) # noqa: B009 + == num_x_separators + ) + assert ( + prompt_token_ids.count(vocab[getattr(hf_processor, "tile_global_token")]) # noqa: B009 == num_y_separators ) diff --git a/tests/models/multimodal/processing/test_minicpmv.py b/tests/models/multimodal/processing/test_minicpmv.py index 59cb4331f3b0..fbcdd77e3f96 100644 --- a/tests/models/multimodal/processing/test_minicpmv.py +++ b/tests/models/multimodal/processing/test_minicpmv.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from vllm.model_executor.models.minicpmv import MiniCPMVProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.parse import ImageSize @@ -21,6 +22,7 @@ def test_get_hf_processor_for_same_model_different_kwargs(model_id: str): ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) info = processor.info + assert isinstance(info, MiniCPMVProcessingInfo) processor_1 = info.get_hf_processor(max_slice_nums=1) processor_2 = info.get_hf_processor(max_slice_nums=2) assert processor_1.image_processor.max_slice_nums == 1 @@ -42,6 +44,7 @@ def test_image_processor_for_different_models(model_ids): limit_mm_per_prompt={"image": 1}, ) first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config) + assert isinstance(first_processor.info, MiniCPMVProcessingInfo) first_image_processor = first_processor.info.get_image_processor() second_ctx = build_model_context( @@ -49,6 +52,7 @@ def test_image_processor_for_different_models(model_ids): limit_mm_per_prompt={"image": 1}, ) second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config) + assert isinstance(second_processor.info, MiniCPMVProcessingInfo) second_image_processor = second_processor.info.get_image_processor() second_processor.info.get_sliced_grid(ImageSize(width=128, height=128)) diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py index 87ca8c842bf1..706737756ff3 100644 --- a/tests/models/multimodal/processing/test_mistral3.py +++ b/tests/models/multimodal/processing/test_mistral3.py @@ -6,7 +6,7 @@ import torch import torch.nn.functional as F from PIL import Image -from transformers import AutoProcessor, BatchFeature +from transformers import AutoProcessor, BatchFeature, PreTrainedConfig from transformers.models.pixtral import PixtralProcessor from vllm.model_executor.layers.fusion.mm_input_norm import ( @@ -79,6 +79,9 @@ def test_pixtral_patch_embed_preserves_image_order_and_strides( reference = [] for image in images: if normalize: + assert isinstance(norm, FusedMMInputNorm) + assert isinstance(norm.weight, torch.Tensor) + assert isinstance(norm.bias, torch.Tensor) normalized = ( image.float() * norm.weight[:, None, None] + norm.bias[:, None, None] ) @@ -250,11 +253,12 @@ def test_mm_device_do_normalize(): normalized_inputs = processor(prompt, mm_items=mm_items) normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] + assert isinstance(normalized_values, list) ctx.model_config.multimodal_config.mm_device_do_normalize = True raw_inputs = processor(prompt, mm_items=mm_items) raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] - assert all(value.dtype == torch.uint8 for value in raw_values) + assert isinstance(raw_values, list) input_norm = build_mm_input_norm(ctx.model_config) patch_size = processor.info.get_hf_config().vision_config.patch_size @@ -271,6 +275,9 @@ def pack_patches(image: torch.Tensor) -> torch.Tensor: ) for raw, normalized in zip(raw_values, normalized_values): + assert isinstance(raw, torch.Tensor) + assert isinstance(normalized, torch.Tensor) + assert raw.dtype == torch.uint8 output = input_norm(pack_patches(raw), normalized.dtype) torch.testing.assert_close( output, pack_patches(normalized), rtol=1e-5, atol=1e-6 @@ -325,6 +332,6 @@ def test_lightonocr_keeps_vision_config_image_size(): encoder_info = processor.info.get_vision_encoder_info() assert isinstance(encoder_info, PixtralHFEncoderInfo) assert not isinstance(encoder_info, Mistral3HFEncoderInfo) - assert encoder_info.get_image_size() == ( - processor.info.get_hf_config().vision_config.image_size - ) + vision_config = processor.info.get_hf_config().vision_config + assert isinstance(vision_config, PreTrainedConfig) + assert encoder_info.get_image_size() == vision_config.image_size diff --git a/tests/models/multimodal/processing/test_mllama4.py b/tests/models/multimodal/processing/test_mllama4.py index 6cf946eff279..362b327f53e7 100644 --- a/tests/models/multimodal/processing/test_mllama4.py +++ b/tests/models/multimodal/processing/test_mllama4.py @@ -3,6 +3,7 @@ """Tests for mllama's multimodal preprocessing and profiling.""" import pytest +import torch from torch import prod from transformers import Llama4Config @@ -36,11 +37,13 @@ def test_profiling(model_id: str, max_model_len: int): tokens_per_patch = ((image_size // patch_size) ** 2) // downsample_ratio mm_data = mm_inputs["mm_kwargs"].get_data() - chunks_per_image = prod(mm_data["patches_per_image"]) + patches_per_image = mm_data["patches_per_image"] + aspect_ratios = mm_data["aspect_ratios"] + assert isinstance(patches_per_image, torch.Tensor) + assert isinstance(aspect_ratios, torch.Tensor) + chunks_per_image = prod(patches_per_image) total_num_patches = chunks_per_image * tokens_per_patch - num_tiles = ( - mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][1] - ) # x-y separator tokens + num_tiles = aspect_ratios[0][0] * aspect_ratios[0][1] # x-y separator tokens total_tokens = ( total_num_patches.item() + num_tiles.item() + 3 ) # image start, image, image end diff --git a/tests/models/multimodal/processing/test_molmo2.py b/tests/models/multimodal/processing/test_molmo2.py index 77c6b8253d4f..e3b30734ed0b 100644 --- a/tests/models/multimodal/processing/test_molmo2.py +++ b/tests/models/multimodal/processing/test_molmo2.py @@ -87,11 +87,15 @@ def test_dummy_video_num_frames_override_honors_min_of_two( seq_len=128, mm_counts={"image": 0, "video": 1}, mm_options=MultiModalDummyOptions( - video=VideoDummyOptions(num_frames=num_frames_override) + video=VideoDummyOptions( + num_frames=num_frames_override, count=999, width=None, height=None + ) ), ) - video, _metadata = data["video"][0] + videos = data["video"] + assert isinstance(videos, list) + video, _metadata = videos[0] assert video.shape[0] == expected_frames diff --git a/tests/models/multimodal/processing/test_moondream3.py b/tests/models/multimodal/processing/test_moondream3.py index a6284ae45fd1..f2c5d967f66d 100644 --- a/tests/models/multimodal/processing/test_moondream3.py +++ b/tests/models/multimodal/processing/test_moondream3.py @@ -98,6 +98,7 @@ def test_processor_pixel_values( # Verify pixel_values shape pixel_values = mm_data_result["pixel_values"] + assert isinstance(pixel_values, torch.Tensor) assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W] assert pixel_values.shape[2] == 3 # RGB channels assert pixel_values.shape[3] == 378 # crop height @@ -413,7 +414,9 @@ def test_processor_apply_accepts_non_pil_image_inputs( assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS mm_kwargs = processed_inputs["mm_kwargs"].get_data() - assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378) + pixel_values = mm_kwargs["pixel_values"] + assert isinstance(pixel_values, torch.Tensor) + assert pixel_values.shape[2:] == (3, 378, 378) class TestMoondream3TilingLogic: diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py index 294d53f23895..c76f1816ad77 100644 --- a/tests/models/multimodal/processing/test_moss_audio.py +++ b/tests/models/multimodal/processing/test_moss_audio.py @@ -34,6 +34,11 @@ from vllm.sequence import IntermediateTensors +def _uninitialized_model() -> MossAudioModel: + # Each test sets only the fields needed by the method under test. + return object.__new__(MossAudioModel) # type: ignore[type-abstract] + + class _Tokenizer: def encode(self, text, **kwargs): return [ord(char) for char in text] @@ -294,7 +299,7 @@ def test_moss_audio_multimodal_processor_handles_token_and_cache_paths(): def test_moss_audio_supports_language_model_lora_only(): assert supports_lora(MossAudioModel) - model = object.__new__(MossAudioModel) + model = _uninitialized_model() assert isinstance(model, SupportsLoRA) mapping = model.get_mm_mapping() @@ -304,7 +309,7 @@ def test_moss_audio_supports_language_model_lora_only(): def test_moss_audio_error_paths(): - model = object.__new__(MossAudioModel) + model = _uninitialized_model() with pytest.raises(ValueError, match="DeepStack audio token count mismatch"): model._cache_deepstack_input_embeds( inputs_embeds=torch.zeros(4, 8), @@ -328,17 +333,17 @@ def test_moss_audio_validates_tp_config(): vllm_config.model_config.hf_config.adapter_hidden_size = 7 with pytest.raises(ValueError, match="adapter_hidden_size"): - MossAudioModel(vllm_config=vllm_config) + MossAudioModel(vllm_config=vllm_config) # type: ignore[abstract] vllm_config = _vllm_config(tensor_parallel_size=2) vllm_config.model_config.hf_config.audio_config.d_model = 6 vllm_config.model_config.hf_config.audio_config.encoder_attention_heads = 3 with pytest.raises(ValueError, match="encoder_attention_heads"): - MossAudioModel(vllm_config=vllm_config) + MossAudioModel(vllm_config=vllm_config) # type: ignore[abstract] def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch(): - model = object.__new__(MossAudioModel) + model = _uninitialized_model() with pytest.raises(ValueError, match="audio_data batch size"): model._parse_and_validate_audio_input( @@ -349,7 +354,7 @@ def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch(): @pytest.mark.parametrize("deepstack_scales", [(), (7, 11)]) def test_moss_audio_embed_multimodal_packs_by_audio(deepstack_scales): - model = object.__new__(MossAudioModel) + model = _uninitialized_model() model.audio_encoder = _FakeAudioEncoder(len(deepstack_scales)) model.audio_adapter = lambda hidden_states: hidden_states * 5 model.deepstack_audio_merger_list = [ @@ -395,7 +400,7 @@ class _FakeLanguageModel: def embed_input_ids(self, input_ids): return torch.zeros(input_ids.shape[0], 8) - model = object.__new__(MossAudioModel) + model = _uninitialized_model() model.language_model = _FakeLanguageModel() model.deepstack_audio_merger_list = [object(), object()] model.deepstack_input_embeds = None @@ -442,7 +447,7 @@ def fake_lm(*args, _calls=calls, **kwargs): return torch.ones(1, 1) _patch_pp_group(monkeypatch, first=first) - model = object.__new__(MossAudioModel) + model = _uninitialized_model() torch.nn.Module.__init__(model) model.language_model = fake_lm cached = IntermediateTensors({"deepstack_input_embeds_0": torch.ones(3, 8)}) @@ -476,7 +481,7 @@ def fake_lm_non_first_rank(*args, **kwargs): return torch.ones(1, 1) _patch_pp_group(monkeypatch, first=False) - model = object.__new__(MossAudioModel) + model = _uninitialized_model() torch.nn.Module.__init__(model) model.language_model = fake_lm_non_first_rank model.deepstack_input_embeds = IntermediateTensors({}) @@ -509,7 +514,7 @@ def make_model(num_layers, deepstack_layers=None): model.start_layer, model.end_layer = 0, num_layers model.layers = torch.nn.ModuleList([AddOne() for _ in range(num_layers)]) model.norm = lambda hidden_states, residual: (hidden_states, residual) - model._maybe_add_hidden_state = lambda aux, *args: aux + monkeypatch.setattr(model, "_maybe_add_hidden_state", lambda aux, *args: aux) model.deepstack_inject_layer_indices = ( range(0) if deepstack_layers is None else deepstack_layers ) @@ -561,7 +566,7 @@ def make_model(num_layers, deepstack_layers=None): } ) ) - language_model = object.__new__(MossQwen3ForCausalLM) + language_model = object.__new__(MossQwen3ForCausalLM) # type: ignore[type-abstract] torch.nn.Module.__init__(language_model) language_model.model = inner_model language_model.config = SimpleNamespace(hidden_size=4) @@ -639,7 +644,7 @@ def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch): assert all(hasattr(attention, name) for name in ("q_proj", "k_proj", "v_proj")) assert hasattr(attention, "out_proj") assert not hasattr(attention, "qkv") - assert attention.k_proj.bias is None + assert getattr(attention, "k_proj").bias is None # noqa: B009 weight_names = [ "layers.0.self_attn.q_proj.weight", diff --git a/tests/models/multimodal/processing/test_moss_transcribe_diarize.py b/tests/models/multimodal/processing/test_moss_transcribe_diarize.py index 5a10e746d2a6..7f888b9446ac 100644 --- a/tests/models/multimodal/processing/test_moss_transcribe_diarize.py +++ b/tests/models/multimodal/processing/test_moss_transcribe_diarize.py @@ -19,9 +19,12 @@ def _question(request_prompt: str = "", hotwords: str | None = None) -> str: hotwords=hotwords, request_prompt=request_prompt, ) - prompt = MossTranscribeDiarizeForConditionalGeneration.get_generation_prompt( + prompt_data = MossTranscribeDiarizeForConditionalGeneration.get_generation_prompt( params - )["prompt"] + ) + assert isinstance(prompt_data, dict) + prompt = prompt_data.get("prompt") + assert isinstance(prompt, str) return prompt.split("<|audio_end|>\n", 1)[1].split("<|im_end|>", 1)[0] diff --git a/tests/models/multimodal/processing/test_nemotron_parse.py b/tests/models/multimodal/processing/test_nemotron_parse.py index 019c06eb15a2..1acbe2eb84c0 100644 --- a/tests/models/multimodal/processing/test_nemotron_parse.py +++ b/tests/models/multimodal/processing/test_nemotron_parse.py @@ -37,12 +37,14 @@ def test_mm_device_do_normalize(width: int, height: int): normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[ "pixel_values" ] + assert isinstance(normalized_values, torch.Tensor) ctx.model_config.multimodal_config.mm_device_do_normalize = True processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[ "pixel_values" ] + assert isinstance(raw_values, torch.Tensor) assert raw_values.dtype == torch.uint8 assert raw_values.shape == normalized_values.shape @@ -71,7 +73,11 @@ def test_array_image_inputs(mm_device_do_normalize: bool, layout: str): def pixel_values(img: object) -> torch.Tensor: mm_items = processor.info.parse_mm_data({"image": [img]}) - return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"] + values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()[ + "pixel_values" + ] + assert isinstance(values, torch.Tensor) + return values torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0) @@ -88,5 +94,6 @@ def test_explicit_kwargs_without_device_normalize(): mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]}) values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"] + assert isinstance(values, torch.Tensor) # The remote processor ignores these kwargs and still normalises on the host. assert values.is_floating_point() diff --git a/tests/models/multimodal/processing/test_nemotron_parse_35.py b/tests/models/multimodal/processing/test_nemotron_parse_35.py index f8562c3e5220..11fbcc1ae45c 100644 --- a/tests/models/multimodal/processing/test_nemotron_parse_35.py +++ b/tests/models/multimodal/processing/test_nemotron_parse_35.py @@ -45,11 +45,15 @@ def test_causal_prefix_preserves_host_normalized_packed_images( mm_items = processor.info.parse_mm_data({"image": [image]}) actual = processor([1], mm_items=mm_items)["mm_kwargs"].get_data() - assert actual["pixel_values"].is_floating_point() - assert actual["pixel_values"].shape[-1] == 3 * 16**2 + pixel_values = actual["pixel_values"] + assert isinstance(pixel_values, torch.Tensor) + assert pixel_values.is_floating_point() + assert pixel_values.shape[-1] == 3 * 16**2 for name in ("pixel_values", "image_sizes", "content_sizes"): + value = actual[name] + assert isinstance(value, torch.Tensor) torch.testing.assert_close( - actual[name], expected[name].to(actual[name].dtype), rtol=0, atol=0 + value, expected[name].to(value.dtype), rtol=0, atol=0 ) @@ -98,4 +102,4 @@ def test_causal_prefix_fields_config_handles_empty_cached_hf_inputs() -> None: fields = processor._get_mm_fields_config(BatchFeature(), {}) - assert fields["pixel_values"].field.slices == [] + assert getattr(fields["pixel_values"].field, "slices") == [] # noqa: B009 diff --git a/tests/models/multimodal/processing/test_nemotron_vl.py b/tests/models/multimodal/processing/test_nemotron_vl.py index b1c1e11892c7..bc2e9782c01e 100644 --- a/tests/models/multimodal/processing/test_nemotron_vl.py +++ b/tests/models/multimodal/processing/test_nemotron_vl.py @@ -5,6 +5,7 @@ from collections.abc import Mapping import pytest +import torch from PIL import Image from transformers import PreTrainedConfig @@ -77,7 +78,9 @@ def _run_check( # Ensure we have the right number of placeholders per num_crops size image_token_id = tokenizer.convert_tokens_to_ids("") img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) - pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape + pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"] + assert isinstance(pixel_values, torch.Tensor) + pixel_shape = pixel_values.shape print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape) assert img_tok_count == 256 * total_expected_num_patches assert pixel_shape[0] == total_expected_num_patches diff --git a/tests/models/multimodal/processing/test_openvla.py b/tests/models/multimodal/processing/test_openvla.py index 36775d53e40f..70744c1f639a 100644 --- a/tests/models/multimodal/processing/test_openvla.py +++ b/tests/models/multimodal/processing/test_openvla.py @@ -164,8 +164,10 @@ def test_openvla_image_processor_outputs_pixel_values() -> None: output = processor(images=image) - assert output["pixel_values"].shape == (1, 6, 224, 224) - assert output["pixel_values"].dtype == torch.float32 + pixel_values = output["pixel_values"] + assert isinstance(pixel_values, torch.Tensor) + assert pixel_values.shape == (1, 6, 224, 224) + assert pixel_values.dtype == torch.float32 def test_openvla_processing_info_token_counts() -> None: @@ -196,6 +198,7 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None: ] == [(1, 1)] assert content.full == [32000] * 256 + assert callable(content.is_embed) is_embed = content.is_embed(content.full) assert is_embed.dtype == torch.bool assert is_embed.tolist() == [True] * 256 diff --git a/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py b/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py index fda9fa7aa8f4..e6c15dd6b383 100644 --- a/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py +++ b/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py @@ -48,7 +48,7 @@ def test_precomputed_vision_embeddings_preserve_items(model_version, modality): torch.nn.Module.__init__(model) model.visual = Mock(dtype=torch.float32, spatial_merge_size=2) embeds = torch.arange(80, dtype=torch.float64).reshape(10, 8) - outputs = model.embed_multimodal( + outputs = getattr(model, "embed_multimodal")( # noqa: B009 **{ f"{modality}_embeds": embeds, f"{modality}_grid_thw": torch.tensor([[1, 2, 4], [2, 4, 4]]), diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 133880b08167..168bf848891d 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -341,9 +341,13 @@ def test_processor_override( # Ensure we have the right number of placeholders per num_crops size hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token) + image_token_id = tokenizer.convert_tokens_to_ids( + getattr(hf_processor, "image_token") # noqa: B009 + ) img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) - pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values"].shape + pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values"] + assert isinstance(pixel_values, torch.Tensor) + pixel_shape = pixel_values.shape assert img_tok_count == expected_toks_per_img * num_imgs assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs @@ -371,6 +375,7 @@ def test_get_image_size_with_most_features( limit_mm_per_prompt={"image": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Qwen2VLProcessingInfo) hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs) merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size @@ -391,7 +396,9 @@ def test_get_image_size_with_most_features( mm_items=processor.info.parse_mm_data(mm_data), hf_processor_mm_kwargs=mm_processor_kwargs, ) - grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist() + image_grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"] + assert isinstance(image_grid_thw, torch.Tensor) + grid_thw = image_grid_thw.tolist() t, h, w = grid_thw[0] tokens = (t * h * w) // (merge_size**2) assert tokens < max_tokens @@ -422,7 +429,9 @@ def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None: for fps in (1.0, 30.0): mm_data = _build_qwen2_5_vl_video_mm_data(num_frames=32, fps=fps) processed = processor(prompt, mm_items=processor.info.parse_mm_data(mm_data)) - spg = processed["mm_kwargs"]["video"][0].get_data()["second_per_grid_ts"] + item = processed["mm_kwargs"]["video"][0] + assert item is not None + spg = item.get_data()["second_per_grid_ts"] assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2) @@ -458,6 +467,7 @@ def test_mm_device_do_normalize( pixel_values_with_normalize = processed_inputs_with_normalize[ "mm_kwargs" ].get_data()["pixel_values"] + assert isinstance(pixel_values_with_normalize, torch.Tensor) dtype = pixel_values_with_normalize.dtype processed_inputs_without_normalize = processor( @@ -468,6 +478,7 @@ def test_mm_device_do_normalize( pixel_values_without_normalize = processed_inputs_without_normalize[ "mm_kwargs" ].get_data()["pixel_values"] + assert isinstance(pixel_values_without_normalize, torch.Tensor) ctx.model_config.multimodal_config.mm_device_do_normalize = True input_norm = build_mm_input_norm(ctx.model_config).to(device) diff --git a/tests/models/multimodal/processing/test_qwen3_omni.py b/tests/models/multimodal/processing/test_qwen3_omni.py index c0ebf0297184..4a458630f24e 100644 --- a/tests/models/multimodal/processing/test_qwen3_omni.py +++ b/tests/models/multimodal/processing/test_qwen3_omni.py @@ -58,7 +58,9 @@ def test_processor_with_audio_sample_rate( # Verify audio tokens are generated hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token) + audio_token_id = tokenizer.convert_tokens_to_ids( + getattr(hf_processor, "audio_token") # noqa: B009 + ) aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id) assert aud_tok_count >= 1, ( @@ -98,7 +100,9 @@ def get_token_count(duration: float) -> int: hf_processor_mm_kwargs=hf_processor_mm_kwargs, ) hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token) + audio_token_id = tokenizer.convert_tokens_to_ids( + getattr(hf_proc, "audio_token") # noqa: B009 + ) return processed["prompt_token_ids"].count(audio_token_id) short_tokens = get_token_count(1.0) diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index 0c74a86ff000..f0c855efa984 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -87,6 +87,7 @@ def test_processor_partial_scoped_image_size(model_id: str) -> None: limit_mm_per_prompt={"image": 1, "video": 0}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Qwen3VLProcessingInfo) image = np.zeros((128, 128, 3), dtype=np.uint8) min_pixels = 256 * 32 * 32 @@ -95,9 +96,11 @@ def test_processor_partial_scoped_image_size(model_id: str) -> None: def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: mm_items = processor.info.parse_mm_data({"image": [image]}) - return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + grid = processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ "image_grid_thw" ] + assert isinstance(grid, torch.Tensor) + return grid stock_grid = process({}) complete_grid = process({"images_kwargs": {"size": complete_size}}) @@ -127,6 +130,7 @@ def test_processor_partial_scoped_video_size(model_id: str) -> None: limit_mm_per_prompt={"image": 0, "video": 1}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, Qwen3VLProcessingInfo) min_pixels = 512 * 32 * 32 default_size = dict(processor.info.get_video_processor().size) @@ -134,9 +138,11 @@ def test_processor_partial_scoped_video_size(model_id: str) -> None: def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8)) - return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + grid = processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ "video_grid_thw" ] + assert isinstance(grid, torch.Tensor) + return grid stock_grid = process({}) complete_grid = process({"videos_kwargs": {"size": complete_size}}) @@ -302,6 +308,7 @@ def _probe_mm_token_budgets( limit_mm_per_prompt={"image": 1, "video": 1}, ) info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info + assert isinstance(info, Qwen3VLProcessingInfo) video = info.get_max_video_tokens( seq_len=500000, mm_counts={"video": 1, "image": 1} ) @@ -381,7 +388,9 @@ def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor: mm_items=processor.info.parse_mm_data({"video": [videos[0]]}), hf_processor_mm_kwargs=hf_kwargs, ) - return single["mm_kwargs"].get_data()["video_grid_thw"][0] + grids = single["mm_kwargs"].get_data()["video_grid_thw"] + assert isinstance(grids, torch.Tensor) + return grids[0] expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs] assert not torch.equal(expected_grids[0], expected_grids[1]) @@ -466,7 +475,9 @@ def test_processor_profiles_fewer_videos_than_configured_options( profiled = processor.get_dummy_mm_inputs({"video": 1}) assert len(profiled["mm_placeholders"]["video"]) == 1 - assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1 + video_grid_thw = profiled["mm_kwargs"].get_data()["video_grid_thw"] + assert isinstance(video_grid_thw, torch.Tensor) + assert video_grid_thw.shape[0] == 1 def _build_video_embeds_mm_data( @@ -584,7 +595,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non capped_dummy = capped.dummy_inputs.get_dummy_mm_data( 1024, {"video": 1}, MultiModalDummyOptions() ) - capped_frames = capped_dummy["video"][0][0].shape[0] + capped_videos = capped_dummy["video"] + assert isinstance(capped_videos, list) + capped_frames = capped_videos[0][0].shape[0] assert capped_frames == 16, ( f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}" ) @@ -598,7 +611,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data( 1024, {"video": 1}, MultiModalDummyOptions() ) - uncapped_frames = uncapped_dummy["video"][0][0].shape[0] + uncapped_videos = uncapped_dummy["video"] + assert isinstance(uncapped_videos, list) + uncapped_frames = uncapped_videos[0][0].shape[0] assert uncapped_frames == 2, ( f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}" ) diff --git a/tests/models/multimodal/processing/test_smolvlm.py b/tests/models/multimodal/processing/test_smolvlm.py index 02181d5d6fb6..4ce67d2cd2a7 100644 --- a/tests/models/multimodal/processing/test_smolvlm.py +++ b/tests/models/multimodal/processing/test_smolvlm.py @@ -5,6 +5,7 @@ import pytest from transformers import SmolVLMConfig +from vllm.model_executor.models.smolvlm import SmolVLMProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from ....conftest import ImageTestAssets @@ -40,6 +41,7 @@ def test_processor_override( limit_mm_per_prompt={"image": num_imgs}, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, SmolVLMProcessingInfo) hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs # Build the image str / prompt based on the number of images we pass @@ -70,7 +72,9 @@ def test_processor_override( processor=hf_processor, mm_kwargs=hf_processor_mm_kwargs, ) - assert num_patches == expected_toks_per_img // hf_processor.image_seq_len + assert num_patches == expected_toks_per_img // getattr( # noqa: B009 + hf_processor, "image_seq_len" + ) hf_processed_inputs = hf_processor( text=prompt, diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index 73be3ab76c0b..17dc63971a36 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -45,11 +45,10 @@ list[Image.Image] | list[np.ndarray] | list[tuple[np.ndarray, dict[str, Any]]] ) AudioInput = list[tuple[np.ndarray, int]] +ResizedData: TypeAlias = Image.Image | np.ndarray | list[Image.Image] -def _resize_data( - _data: Image.Image | np.ndarray, size_factor: float -) -> Image.Image | np.ndarray: +def _resize_data(_data: ResizedData, size_factor: float) -> ResizedData: assert size_factor <= 1, "Size factor must be less than 1" # Image input if isinstance(_data, Image.Image): @@ -75,7 +74,7 @@ def _resize_data( def resize_mm_data( data: ImageInput | VideoInput | AudioInput, size_factors: tuple[float, ...] -) -> ImageInput | VideoInput | AudioInput: +) -> list[ResizedData]: size_factors = size_factors[: len(data)] if is_list_of(data, (Image.Image, np.ndarray, list)): return [_resize_data(d, s) for d, s in zip(data, size_factors)] @@ -247,11 +246,13 @@ def test_model_tensor_schema(model_id: str): def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions: if modality == "video": - return VideoDummyOptions(count=count) + return VideoDummyOptions( + count=count, num_frames=None, width=None, height=None + ) if modality == "image": - return ImageDummyOptions(count=count) + return ImageDummyOptions(count=count, width=None, height=None) if modality == "audio": - return AudioDummyOptions(count=count) + return AudioDummyOptions(count=count, length=None) return BaseDummyOptions(count=count) model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions( diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py index 1c1204d8e5ce..8c567753987e 100644 --- a/tests/models/multimodal/processing/test_transformers_audio.py +++ b/tests/models/multimodal/processing/test_transformers_audio.py @@ -4,6 +4,9 @@ import pytest from vllm.config import ModelConfig +from vllm.model_executor.models.transformers.multimodal import ( + MultiModalProcessor as TransformersMultiModalProcessor, +) from vllm.multimodal import MULTIMODAL_REGISTRY AUDIO_MODEL_SETTINGS = { @@ -103,6 +106,7 @@ def test_audio_multimodal_processor(model_id): audio_items = result.get("mm_kwargs", {}).get("audio", []) assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}" + assert audio_items[0] is not None item_keys = list(audio_items[0].keys()) has_features = "input_features" in item_keys or "input_values" in item_keys assert has_features, ( @@ -119,7 +123,7 @@ def test_audio_multiple_inputs(separator): ModelConfig(model=model_id, model_impl="transformers") ) - audio_token = mm_processor.info.get_hf_processor().audio_token + audio_token = getattr(mm_processor.info.get_hf_processor(), "audio_token") # noqa: B009 # One token per audio; the processor expands each to its placeholder run. prompt = ( "<|start_of_role|>user<|end_of_role|>" @@ -142,6 +146,7 @@ def test_audio_fields_not_claimed_by_image(): model_id = "ibm-granite/granite-speech-3.3-2b" model_config = ModelConfig(model=model_id, model_impl="transformers") mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + assert isinstance(mm_processor, TransformersMultiModalProcessor) audio_keys = ["input_features", "input_features_mask"] owned = mm_processor._partition_keys_by_modality(audio_keys, ["audio", "image"]) @@ -155,6 +160,7 @@ def test_unclaimed_fields_warn_rather_than_raise(): model_id = "ibm-granite/granite-speech-3.3-2b" model_config = ModelConfig(model=model_id, model_impl="transformers") mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + assert isinstance(mm_processor, TransformersMultiModalProcessor) owned = mm_processor._partition_keys_by_modality( ["input_features", "surprise_field"], ["audio", "image"] diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py index d45912a4b141..61b6720e5967 100644 --- a/tests/models/multimodal/processing/test_transformers_image.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -3,6 +3,7 @@ from unittest.mock import patch import pytest +import torch from vllm.assets.image import ImageAsset from vllm.config import ModelConfig @@ -98,7 +99,7 @@ def test_batch_padding_removed_from_image_items(): mm_processor = MULTIMODAL_REGISTRY.create_processor( ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers") ) - image_token = mm_processor.info.get_hf_processor().image_token + image_token = getattr(mm_processor.info.get_hf_processor(), "image_token") # noqa: B009 images = [ ImageAsset("cherry_blossom").pil_image, @@ -113,8 +114,12 @@ def test_batch_padding_removed_from_image_items(): items = result["mm_kwargs"]["image"] shapes = set() for item in items: - height, width = item["image_sizes"].data.flatten().tolist() + assert item is not None + image_sizes = item["image_sizes"].data + assert isinstance(image_sizes, torch.Tensor) + height, width = image_sizes.flatten().tolist() pixel_values = item["pixel_values"].data + assert isinstance(pixel_values, torch.Tensor) assert tuple(pixel_values.shape[-2:]) == (height, width) shapes.add(tuple(pixel_values.shape)) @@ -126,7 +131,7 @@ def _process_one_gemma3_image(): mm_processor = MULTIMODAL_REGISTRY.create_processor( ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers") ) - boi_token = mm_processor.info.get_hf_processor().boi_token + boi_token = getattr(mm_processor.info.get_hf_processor(), "boi_token") # noqa: B009 return mm_processor( prompt=f"{boi_token} What is this?", mm_items=mm_processor.info.parse_mm_data( @@ -219,6 +224,7 @@ def test_repeated_image_hits_the_processor_cache(): model_config = ModelConfig( model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers" ) + assert model_config.multimodal_config is not None model_config.multimodal_config.mm_processor_cache_gb = 4 mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) cache = MultiModalProcessorOnlyCache(model_config) @@ -303,5 +309,9 @@ def test_nested_image_fields_split_per_image(): items = result["mm_kwargs"]["image"] assert len(items) == 2 for item in items: + assert item is not None pixel_values = item["pixel_values"].data - assert pixel_values.shape[1] == int(item["num_image_patches"].data) + num_image_patches = item["num_image_patches"].data + assert isinstance(pixel_values, torch.Tensor) + assert isinstance(num_image_patches, torch.Tensor) + assert pixel_values.shape[1] == int(num_image_patches) diff --git a/tests/models/multimodal/processing/test_transformers_video.py b/tests/models/multimodal/processing/test_transformers_video.py index 8d06843f06d3..b454e96f179a 100644 --- a/tests/models/multimodal/processing/test_transformers_video.py +++ b/tests/models/multimodal/processing/test_transformers_video.py @@ -2,9 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import numpy as np import pytest +import torch from transformers import Qwen3VLVideoProcessor from vllm.config import ModelConfig +from vllm.model_executor.models.transformers.multimodal import MultiModalProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache @@ -94,7 +96,7 @@ def test_video_multimodal_processor(model_id): assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist() (placeholder,) = result["mm_placeholders"]["video"] - (num_video_tokens,) = hf_processor._get_num_multimodal_tokens( + (num_video_tokens,) = getattr(hf_processor, "_get_num_multimodal_tokens")( # noqa: B009 video_sizes=[frames.shape[:3]] )["num_video_tokens"] assert placeholder.get_num_embeds() == num_video_tokens @@ -103,9 +105,14 @@ def test_video_multimodal_processor(model_id): ) (item,) = result["mm_kwargs"]["video"] - rows = item["pixel_values_videos"].data.shape[0] + assert item is not None + pixel_values_videos = item["pixel_values_videos"].data + assert isinstance(pixel_values_videos, torch.Tensor) + rows = pixel_values_videos.shape[0] if "video_grid_thw" in item: - assert rows == int(item["num_video_patches"].data) + num_video_patches = item["num_video_patches"].data + assert isinstance(num_video_patches, torch.Tensor) + assert rows == int(num_video_patches) else: assert rows == len(frames) @@ -168,6 +175,8 @@ def test_video_fields_not_claimed_by_image(): (image_item,) = result["mm_kwargs"]["image"] (video_item,) = result["mm_kwargs"]["video"] + assert image_item is not None + assert video_item is not None assert {"pixel_values", "image_grid_thw"} <= set(image_item.keys()) assert {"pixel_values_videos", "video_grid_thw"} <= set(video_item.keys()) assert len(result["mm_placeholders"]["image"]) == 1 @@ -179,6 +188,7 @@ def test_repeated_video_hits_the_processor_cache(): model_config = ModelConfig( model="Qwen/Qwen3-VL-2B-Instruct", model_impl="transformers" ) + assert model_config.multimodal_config is not None model_config.multimodal_config.mm_processor_cache_gb = 4 mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) cache = MultiModalProcessorOnlyCache(model_config) @@ -222,7 +232,13 @@ def test_merged_token_fields_split_per_video(): ) items = result["mm_kwargs"]["video"] - assert [len(item["video_compression_mask"].data) for item in items] == [128, 64] + assert all(item is not None for item in items) + assert [ + len(item["video_compression_mask"].data) for item in items if item is not None + ] == [ + 128, + 64, + ] def test_video_sampled_by_processor(): @@ -262,6 +278,7 @@ def test_dummy_video_has_the_most_tokens(model_id): ModelConfig(model=model_id, model_impl="transformers") ) info = mm_processor.info + assert isinstance(info, MultiModalProcessingInfo) seq_len = info.ctx.model_config.max_model_len result = mm_processor.get_dummy_mm_inputs({"video": 1}) @@ -271,7 +288,7 @@ def test_dummy_video_has_the_most_tokens(model_id): assert placeholder.get_num_embeds() == max_tokens # Shrinking the dummy frames must never cost tokens num_frames = info.get_num_frames_with_most_features(seq_len, {"video": 1}) - (huge_frame_tokens,) = info.get_hf_processor()._get_num_multimodal_tokens( - video_sizes=[[num_frames, 10_000, 10_000]] - )["num_video_tokens"] + (huge_frame_tokens,) = getattr( # noqa: B009 + info.get_hf_processor(), "_get_num_multimodal_tokens" + )(video_sizes=[[num_frames, 10_000, 10_000]])["num_video_tokens"] assert max_tokens >= huge_frame_tokens diff --git a/tests/models/multimodal/processing/test_unlimited_ocr.py b/tests/models/multimodal/processing/test_unlimited_ocr.py index 87d2f2763119..06f68f332cd5 100644 --- a/tests/models/multimodal/processing/test_unlimited_ocr.py +++ b/tests/models/multimodal/processing/test_unlimited_ocr.py @@ -6,6 +6,7 @@ from PIL import Image from vllm.config import ModelConfig +from vllm.model_executor.models.unlimited_ocr import UnlimitedOCRProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.processing.context import TimingContext @@ -24,16 +25,17 @@ def test_profiling_tokens_cover_all_crop_grids(): model_config_kwargs={"max_model_len": 8192}, ) mm_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(mm_processor.info, UnlimitedOCRProcessingInfo) hf_processor = mm_processor.info.get_hf_processor() dummy_inputs = mm_processor.get_dummy_mm_inputs({"image": 1}) budget = dummy_inputs["mm_placeholders"]["image"][0].get_num_embeds() for width_tiles, height_tiles in calculate_aspect_ratios( - max_num=hf_processor.max_crops + max_num=getattr(hf_processor, "max_crops") # noqa: B009 ): num_tokens = mm_processor.info.get_num_image_tokens( - image_width=width_tiles * hf_processor.image_size, - image_height=height_tiles * hf_processor.image_size, + image_width=width_tiles * getattr(hf_processor, "image_size"), # noqa: B009 + image_height=height_tiles * getattr(hf_processor, "image_size"), # noqa: B009 ) assert num_tokens <= budget, (width_tiles, height_tiles, num_tokens, budget) @@ -48,6 +50,7 @@ def test_image_cache_identity_across_image_counts(use_uuid: bool, use_embeds: bo ) processor = MULTIMODAL_REGISTRY.create_processor(config) tokenizer = processor.info.get_tokenizer() + images: list[torch.Tensor] | list[Image.Image] if use_embeds: images = [torch.ones(8, config.get_hidden_size())] * 3 else: diff --git a/tests/models/multimodal/processing/test_whisper.py b/tests/models/multimodal/processing/test_whisper.py index 1627b48914c3..e7271fa92e5e 100644 --- a/tests/models/multimodal/processing/test_whisper.py +++ b/tests/models/multimodal/processing/test_whisper.py @@ -4,7 +4,9 @@ import numpy as np import pytest +import torch +from vllm.model_executor.models.whisper import WhisperProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from ...utils import build_model_context @@ -21,6 +23,7 @@ def test_audio_features_fit_encoder_window( """ ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": 1}) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert isinstance(processor.info, WhisperProcessingInfo) feature_extractor = processor.info.get_feature_extractor() sampling_rate = feature_extractor.sampling_rate @@ -34,4 +37,5 @@ def test_audio_features_fit_encoder_window( ) input_features = processed_inputs["mm_kwargs"].get_data()["input_features"] + assert isinstance(input_features, torch.Tensor) assert input_features.shape[-1] == feature_extractor.nb_max_frames diff --git a/tests/models/multimodal/test_mapping.py b/tests/models/multimodal/test_mapping.py index a213091e3d09..ec92ff9eb854 100644 --- a/tests/models/multimodal/test_mapping.py +++ b/tests/models/multimodal/test_mapping.py @@ -342,7 +342,7 @@ def test_hf_model_weights_mapper(model_arch: str): hf_dummy_model = create_dummy_model(model_id, model_arch) hf_converted_weights = hf_dummy_model.named_parameters() hf_converted_buffers = hf_dummy_model.named_buffers() - mapper: WeightsMapper = model_cls.hf_to_vllm_mapper + mapper: WeightsMapper = getattr(model_cls, "hf_to_vllm_mapper") # noqa: B009 mapped_original_weights = mapper.apply(original_weights) mapped_hf_converted_weights = mapper.apply(hf_converted_weights) @@ -363,8 +363,7 @@ def test_hf_model_weights_mapper(model_arch: str): config = hf_dummy_model.config key = "tie_word_embeddings" if getattr(config.get_text_config(), key, False) or getattr(config, key, False): - mapped_tied_weights = mapper.apply((k, None) for k in tied) - tied_weight_names = set(map(lambda x: x[0], mapped_tied_weights)) + tied_weight_names = set(mapper.apply_list(tied)) ref_weight_names -= tied_weight_names weights_missing = ref_weight_names - weight_names diff --git a/tests/models/multimodal/test_mimo_v2_omni.py b/tests/models/multimodal/test_mimo_v2_omni.py index deccace50d4e..9267217c278c 100644 --- a/tests/models/multimodal/test_mimo_v2_omni.py +++ b/tests/models/multimodal/test_mimo_v2_omni.py @@ -58,10 +58,9 @@ def test_window_attention_applies_sinks(vision_attn_env, num_kv_heads): attn.sinks.data.normal_() total = sum(SEQ_LENS) - opts = dict(device="cuda", dtype=torch.bfloat16) - q = torch.randn(total, NUM_HEADS, HEAD_DIM, **opts) - k = torch.randn(total, num_kv_heads, HEAD_DIM, **opts) - v = torch.randn(total, num_kv_heads, HEAD_DIM, **opts) + q = torch.randn(total, NUM_HEADS, HEAD_DIM, device="cuda", dtype=torch.bfloat16) + k = torch.randn(total, num_kv_heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16) + v = torch.randn(total, num_kv_heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16) cu_seqlens = torch.tensor( [0, *torch.tensor(SEQ_LENS).cumsum(0).tolist()], device="cuda", diff --git a/tests/models/multimodal/test_nano_nemotron_vl.py b/tests/models/multimodal/test_nano_nemotron_vl.py index 41f9428e6d73..9fc8c822a1fe 100644 --- a/tests/models/multimodal/test_nano_nemotron_vl.py +++ b/tests/models/multimodal/test_nano_nemotron_vl.py @@ -19,9 +19,13 @@ ) +def _uninitialized_model() -> NemotronH_Nano_VL_V2: + return object.__new__(NemotronH_Nano_VL_V2) # type: ignore[type-abstract] + + @pytest.mark.parametrize("input_key", ["image_embeds", "video_embeds"]) def test_precomputed_multimodal_embeddings(input_key: str): - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() embeds = torch.randn(2, 4, 8) outputs = model.embed_multimodal(**{input_key: embeds}) @@ -77,21 +81,8 @@ def load_weights(self, weights): self.loaded_weights = list(weights) -class _FakeTensor: - """Sentinel stand-in for torch.Tensor in load_weights tests. Supports the - .detach().clone() chain used by load_weights for buffered mm weights; - both methods return self so identity (and the existing equality - assertions) are preserved through cloning.""" - - def detach(self): - return self - - def clone(self): - return self - - def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode(): - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() language_model = _LanguageModel() object.__setattr__(model, "model_config", _ModelConfig()) object.__setattr__(model, "language_model", language_model) @@ -100,15 +91,15 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode(): object.__setattr__(model, "vision_final_layernorm", _MissingMultiModalModule()) object.__setattr__(model, "sound_encoder", None) - language_weight = object() + language_weight = torch.tensor(1.0) model.load_weights( [ ("language_model.layers.0.weight", language_weight), - ("mlp1.0.weight", object()), - ("vision_model.radio_model.encoder.weight", object()), - ("sound_encoder.encoder.weight", object()), - ("vision_projector.vision_final_layernorm.weight", object()), - ("vision_projector.vision_final_layernorm.bias", object()), + ("mlp1.0.weight", torch.empty(0)), + ("vision_model.radio_model.encoder.weight", torch.empty(0)), + ("sound_encoder.encoder.weight", torch.empty(0)), + ("vision_projector.vision_final_layernorm.weight", torch.empty(0)), + ("vision_projector.vision_final_layernorm.bias", torch.empty(0)), ] ) @@ -116,7 +107,7 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode(): def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder(): - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() language_model = _LanguageModel() vision_model = _VisionModel() object.__setattr__(model, "model_config", _ImageOnlyModelConfig()) @@ -126,8 +117,8 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder(): object.__setattr__(model, "vision_final_layernorm", None) object.__setattr__(model, "sound_encoder", None) - language_weight = object() - vision_weight = _FakeTensor() + language_weight = torch.tensor(1.0) + vision_weight = torch.tensor([2.0, 3.0]) model.load_weights( [ ("language_model.layers.0.weight", language_weight), @@ -136,13 +127,16 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder(): ) assert language_model.loaded_weights == [("layers.0.weight", language_weight)] - assert vision_model.loaded_weights == [ - ("radio_model.encoder.weight", vision_weight) - ] + assert len(vision_model.loaded_weights) == 1 + name, loaded_weight = vision_model.loaded_weights[0] + assert name == "radio_model.encoder.weight" + assert isinstance(loaded_weight, torch.Tensor) + torch.testing.assert_close(loaded_weight, vision_weight) + assert loaded_weight is not vision_weight def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights(): - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() language_model = _LanguageModel() vision_model = _VisionModel() object.__setattr__(model, "model_config", _ImageOnlyModelConfig()) @@ -153,11 +147,11 @@ def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights(): object.__setattr__(model, "sound_encoder", None) with pytest.raises(AssertionError): - model.load_weights([("sound_encoder.encoder.weight", object())]) + model.load_weights([("sound_encoder.encoder.weight", torch.empty(0))]) def _vision_norm_model(use_layernorm=True): - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() torch.nn.Module.__init__(model) model.model_config = _ImageOnlyModelConfig() model.language_model = _LanguageModel() diff --git a/tests/models/multimodal/test_openpangu_vl.py b/tests/models/multimodal/test_openpangu_vl.py index e8bab16d5d96..7267e893bcc4 100644 --- a/tests/models/multimodal/test_openpangu_vl.py +++ b/tests/models/multimodal/test_openpangu_vl.py @@ -22,14 +22,13 @@ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: class _OpenPanguVL(OpenPanguVLForConditionalGeneration): + language_model: _LanguageModel + def __init__(self, hidden_size: int) -> None: nn.Module.__init__(self) self.config = SimpleNamespace(image_token_id=41, video_token_id=42) self.language_model = _LanguageModel(hidden_size) - def get_language_model(self) -> _LanguageModel: - return self.language_model - @pytest.mark.parametrize("device_type", ["cpu", "cuda"]) def test_get_input_embeddings_merges_image_and_video_embeddings( @@ -40,7 +39,7 @@ def test_get_input_embeddings_merges_image_and_video_embeddings( device = torch.device(device_type) hidden_size = 4 - model = _OpenPanguVL(hidden_size).to(device) + model = _OpenPanguVL(hidden_size).to(device) # type: ignore[abstract] input_ids = torch.tensor([7, 41, 41, 8, 42, 9], device=device) multimodal_embeddings = ( torch.full((2, hidden_size), -1.0, device=device), diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 5c90eee54963..66d4e61f0011 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -50,6 +50,7 @@ "tests/lora", "tests/model_executor", "tests/models/language", + "tests/models/multimodal", "tests/models/quantization", "tests/models/test_initialization.py", "tests/multimodal", @@ -95,7 +96,6 @@ "tests/compile", "tests/distributed", "tests/models", - "tests/models/multimodal", "tests/v1", "tests/v1/core", "tests/v1/engine", From 7c33975d305654e00a27243a2deaf86df364e208 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:39:19 +0000 Subject: [PATCH 2/6] Fix mypy errors at source and drop unneeded test workarounds - Pass DummyOptions defaults to Field by keyword so mypy sees them as optional - Give SupportsLoRA.lora_manager a None default so explicit subclasses are not implicitly abstract - Revert getattr/noqa B009, type: ignore[abstract] and dict() workarounds that mypy does not need Co-Authored-By: Claude Opus 5.5 Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- .../multimodal/generation/test_mm_prefix_lm.py | 2 +- .../generation/test_nemotron_parse_35.py | 2 +- .../multimodal/generation/test_vit_cudagraph.py | 2 +- .../multimodal/generation/test_voxtral_realtime.py | 2 +- .../multimodal/generation/vlm_utils/model_utils.py | 2 +- tests/models/multimodal/pooling/test_phi3v.py | 2 +- tests/models/multimodal/processing/test_common.py | 8 +++----- tests/models/multimodal/processing/test_gemma3.py | 2 +- tests/models/multimodal/processing/test_gemma4.py | 2 +- .../multimodal/processing/test_gemma4_unified.py | 6 +++--- tests/models/multimodal/processing/test_glm4_1v.py | 2 +- .../multimodal/processing/test_lightonocr.py | 8 ++++---- tests/models/multimodal/processing/test_llama4.py | 11 +++-------- tests/models/multimodal/processing/test_molmo2.py | 4 +--- .../multimodal/processing/test_moss_audio.py | 8 ++++---- .../models/multimodal/processing/test_qwen2_vl.py | 4 +--- .../multimodal/processing/test_qwen3_omni.py | 8 ++------ tests/models/multimodal/processing/test_smolvlm.py | 4 +--- .../multimodal/processing/test_tensor_schema.py | 8 +++----- .../processing/test_transformers_audio.py | 2 +- .../processing/test_transformers_image.py | 4 ++-- .../processing/test_transformers_video.py | 8 ++++---- .../multimodal/processing/test_unlimited_ocr.py | 6 +++--- vllm/config/multimodal.py | 14 +++++++------- vllm/model_executor/models/interfaces.py | 2 +- 25 files changed, 52 insertions(+), 71 deletions(-) diff --git a/tests/models/multimodal/generation/test_mm_prefix_lm.py b/tests/models/multimodal/generation/test_mm_prefix_lm.py index e1e3e1223266..10ff63f6c0fe 100644 --- a/tests/models/multimodal/generation/test_mm_prefix_lm.py +++ b/tests/models/multimodal/generation/test_mm_prefix_lm.py @@ -50,7 +50,7 @@ def _get_hf_prefill_hidden(hf_model: HfRunner, image: Any): inputs = hf_model.get_inputs([PROMPT], images=[image])[0] with torch.no_grad(): outputs = hf_model.model.model( - **hf_model.wrap_device(dict(inputs)), + **hf_model.wrap_device(inputs), use_cache=False, ) return outputs.last_hidden_state[0].detach().float().cpu() diff --git a/tests/models/multimodal/generation/test_nemotron_parse_35.py b/tests/models/multimodal/generation/test_nemotron_parse_35.py index e151e184f478..f5b9884df4c7 100644 --- a/tests/models/multimodal/generation/test_nemotron_parse_35.py +++ b/tests/models/multimodal/generation/test_nemotron_parse_35.py @@ -35,7 +35,7 @@ def generate_hf_logprobs_from_logits( outputs = [] for model_inputs in inputs: generated = hf_model.model.generate( - **hf_model.wrap_device(dict(model_inputs)), + **hf_model.wrap_device(model_inputs), do_sample=False, max_new_tokens=max_tokens, output_logits=True, diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index 6db55fa29d83..8751a94fe690 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -577,7 +577,7 @@ def _check_eonly_encoder_outputs(worker, batches): for batch in batches: kwargs = {key: value.to(runner.device) for key, value in batch.items()} actual = manager.execute(kwargs) - expected = getattr(runner.model, "embed_multimodal")(**kwargs) # noqa: B009 + expected = runner.model.embed_multimodal(**kwargs) assert len(actual) == len(expected) for output, reference in zip(actual, expected): assert torch.isfinite(output).all() diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index 719ff4183b12..e05930b0f36b 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -138,7 +138,7 @@ async def async_engine(request): def _build_inputs( audio_assets: list[AudioAsset], tokenizer: MistralTokenizer ) -> tuple[list[dict], list[SamplingParams]]: - audio_config = getattr(tokenizer.instruct_tokenizer.tokenizer, "audio") # noqa: B009 + audio_config = tokenizer.instruct_tokenizer.tokenizer.audio inputs = [] sampling_params = [] for audio_asset in audio_assets: diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 4b7949d07c5a..f201d5cc2137 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -836,7 +836,7 @@ def _internvl_generate( if not isinstance(self.language_model, GenerationMixin): pytest.skip("HF impl is not compatible with current transformers") - outputs = getattr(self.language_model, "generate")( # noqa: B009 + outputs = self.language_model.generate( **forward_kwargs, **generate_kwargs, ) diff --git a/tests/models/multimodal/pooling/test_phi3v.py b/tests/models/multimodal/pooling/test_phi3v.py index 7546b04f06c2..246f6e383fb1 100644 --- a/tests/models/multimodal/pooling/test_phi3v.py +++ b/tests/models/multimodal/pooling/test_phi3v.py @@ -192,7 +192,7 @@ def test_models_image_special_tokens_processing( return_tensors="pt", ) - image_token_id = getattr(hf_processor, "get_special_image_token_id")() # noqa: B009 + image_token_id = hf_processor.get_special_image_token_id() hf_prompt_token_ids = [ image_token_id if token_id < 0 else token_id for token_id in hf_inputs["input_ids"][0].tolist() diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index f830843d0dda..7216c47bd347 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -356,13 +356,11 @@ def _test_processing_correctness( def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions: if modality == "video": - return VideoDummyOptions( - count=count, num_frames=None, width=None, height=None - ) + return VideoDummyOptions(count=count) if modality == "image": - return ImageDummyOptions(count=count, width=None, height=None) + return ImageDummyOptions(count=count) if modality == "audio": - return AudioDummyOptions(count=count, length=None) + return AudioDummyOptions(count=count) return BaseDummyOptions(count=count) # Assign normalized DummyOptions to the model config diff --git a/tests/models/multimodal/processing/test_gemma3.py b/tests/models/multimodal/processing/test_gemma3.py index 114799b068f5..8b61deb91d6f 100644 --- a/tests/models/multimodal/processing/test_gemma3.py +++ b/tests/models/multimodal/processing/test_gemma3.py @@ -176,7 +176,7 @@ def test_get_image_size_with_most_features( ) prompt = "" - image_seq_length = getattr(hf_processor, "image_seq_length") # noqa: B009 + image_seq_length = hf_processor.image_seq_length for asset in image_assets: mm_data = {"image": [asset.pil_image]} diff --git a/tests/models/multimodal/processing/test_gemma4.py b/tests/models/multimodal/processing/test_gemma4.py index c2ba4a75b342..90f5d0e24eba 100644 --- a/tests/models/multimodal/processing/test_gemma4.py +++ b/tests/models/multimodal/processing/test_gemma4.py @@ -281,7 +281,7 @@ def test_video_cache_is_independent_of_image_kwargs( def process(mm_kwargs, cache): return processor( - getattr(hf_processor, "image_token") + getattr(hf_processor, "video_token"), # noqa: B009 + hf_processor.image_token + hf_processor.video_token, mm_items, mm_uuid_items={"video": [video_uuid]}, hf_processor_mm_kwargs=mm_kwargs, diff --git a/tests/models/multimodal/processing/test_gemma4_unified.py b/tests/models/multimodal/processing/test_gemma4_unified.py index bd5a15c91fb2..3754e4abc14f 100644 --- a/tests/models/multimodal/processing/test_gemma4_unified.py +++ b/tests/models/multimodal/processing/test_gemma4_unified.py @@ -407,7 +407,7 @@ def test_process_image_input_batching_and_chunking(): """ from unittest.mock import patch - model = _MockUnifiedModel() # type: ignore[abstract] + model = _MockUnifiedModel() # 1. 3D stacked image tensor pv = torch.randn(2, 5, 48) @@ -446,7 +446,7 @@ def test_process_image_input_batching_and_chunking(): def test_process_video_input_batching(): """Verify _process_video_input with frame counts and lists of tensors.""" - model = _MockUnifiedModel() # type: ignore[abstract] + model = _MockUnifiedModel() # 2 videos: 3 frames and 2 frames, each frame has 4 patches pv_vid = torch.randn(5, 4, 48) @@ -487,7 +487,7 @@ def test_process_video_input_batching(): def test_cuda_execution_with_gpu_sync_allowed(): """Verify vision processing on CUDA under gpu_sync_allowed.""" device = "cuda:0" - model = _MockUnifiedModel(device=device, dtype=torch.bfloat16) # type: ignore[abstract] + model = _MockUnifiedModel(device=device, dtype=torch.bfloat16) pv = torch.randn(2, 5, 48, device=device, dtype=torch.bfloat16) pp = torch.randint(0, 100, (2, 5, 2), device=device) diff --git a/tests/models/multimodal/processing/test_glm4_1v.py b/tests/models/multimodal/processing/test_glm4_1v.py index 18cd4c634aad..adf7219db1b7 100644 --- a/tests/models/multimodal/processing/test_glm4_1v.py +++ b/tests/models/multimodal/processing/test_glm4_1v.py @@ -136,7 +136,7 @@ def test_processor_override( # Ensure we have the right number of placeholders per num_crops size hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token_id = getattr(hf_processor, "image_token_id") # noqa: B009 + image_token_id = hf_processor.image_token_id video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0] diff --git a/tests/models/multimodal/processing/test_lightonocr.py b/tests/models/multimodal/processing/test_lightonocr.py index 34479ed47671..c138808c7736 100644 --- a/tests/models/multimodal/processing/test_lightonocr.py +++ b/tests/models/multimodal/processing/test_lightonocr.py @@ -35,16 +35,16 @@ def test_lightonocr_image_embeddings_preserve_lengths_and_values( for i, size in enumerate(feature_sizes) ] image_input = torch.stack(embeddings) if tensor_input else embeddings - prompt = " ".join([getattr(hf_processor, "image_token")] * len(embeddings)) # noqa: B009 + prompt = " ".join([hf_processor.image_token] * len(embeddings)) if prompt_as_tokens: prompt = ctx.get_tokenizer().encode(prompt) items = processor.info.parse_mm_data({"image": image_input}) processed = processor(prompt, mm_items=items) - assert processed["prompt_token_ids"].count( - getattr(hf_processor, "image_token_id") # noqa: B009 - ) == sum(feature_sizes) + assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum( + feature_sizes + ) assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes for item, expected in zip(processed["mm_kwargs"]["image"], embeddings): assert item is not None diff --git a/tests/models/multimodal/processing/test_llama4.py b/tests/models/multimodal/processing/test_llama4.py index e4743396c869..e3d4a94796b5 100644 --- a/tests/models/multimodal/processing/test_llama4.py +++ b/tests/models/multimodal/processing/test_llama4.py @@ -63,9 +63,7 @@ def test_processor_override( prompt_token_ids = processed_inputs["prompt_token_ids"] assert prompt_token_ids.count(config.boi_token_index) == num_imgs assert prompt_token_ids.count(config.eoi_token_index) == num_imgs - assert ( - prompt_token_ids.count(vocab[getattr(hf_processor, "image_token")]) == num_imgs # noqa: B009 - ) + assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs aspect_ratios = mm_data["aspect_ratios"] assert isinstance(aspect_ratios, torch.Tensor) num_x_separators = num_y_separators = 0 @@ -73,12 +71,9 @@ def test_processor_override( if tiles_x * tiles_y > 1: num_x_separators += (tiles_x - 1) * tiles_y num_y_separators += tiles_y + assert prompt_token_ids.count(vocab[hf_processor.tile_token]) == num_x_separators assert ( - prompt_token_ids.count(vocab[getattr(hf_processor, "tile_token")]) # noqa: B009 - == num_x_separators - ) - assert ( - prompt_token_ids.count(vocab[getattr(hf_processor, "tile_global_token")]) # noqa: B009 + prompt_token_ids.count(vocab[hf_processor.tile_global_token]) == num_y_separators ) diff --git a/tests/models/multimodal/processing/test_molmo2.py b/tests/models/multimodal/processing/test_molmo2.py index e3b30734ed0b..1ecefb23a310 100644 --- a/tests/models/multimodal/processing/test_molmo2.py +++ b/tests/models/multimodal/processing/test_molmo2.py @@ -87,9 +87,7 @@ def test_dummy_video_num_frames_override_honors_min_of_two( seq_len=128, mm_counts={"image": 0, "video": 1}, mm_options=MultiModalDummyOptions( - video=VideoDummyOptions( - num_frames=num_frames_override, count=999, width=None, height=None - ) + video=VideoDummyOptions(num_frames=num_frames_override) ), ) diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py index c76f1816ad77..369f005dcc2d 100644 --- a/tests/models/multimodal/processing/test_moss_audio.py +++ b/tests/models/multimodal/processing/test_moss_audio.py @@ -333,13 +333,13 @@ def test_moss_audio_validates_tp_config(): vllm_config.model_config.hf_config.adapter_hidden_size = 7 with pytest.raises(ValueError, match="adapter_hidden_size"): - MossAudioModel(vllm_config=vllm_config) # type: ignore[abstract] + MossAudioModel(vllm_config=vllm_config) vllm_config = _vllm_config(tensor_parallel_size=2) vllm_config.model_config.hf_config.audio_config.d_model = 6 vllm_config.model_config.hf_config.audio_config.encoder_attention_heads = 3 with pytest.raises(ValueError, match="encoder_attention_heads"): - MossAudioModel(vllm_config=vllm_config) # type: ignore[abstract] + MossAudioModel(vllm_config=vllm_config) def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch(): @@ -566,7 +566,7 @@ def make_model(num_layers, deepstack_layers=None): } ) ) - language_model = object.__new__(MossQwen3ForCausalLM) # type: ignore[type-abstract] + language_model = object.__new__(MossQwen3ForCausalLM) torch.nn.Module.__init__(language_model) language_model.model = inner_model language_model.config = SimpleNamespace(hidden_size=4) @@ -644,7 +644,7 @@ def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch): assert all(hasattr(attention, name) for name in ("q_proj", "k_proj", "v_proj")) assert hasattr(attention, "out_proj") assert not hasattr(attention, "qkv") - assert getattr(attention, "k_proj").bias is None # noqa: B009 + assert attention.k_proj.bias is None weight_names = [ "layers.0.self_attn.q_proj.weight", diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 168bf848891d..21f45d1f09dd 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -341,9 +341,7 @@ def test_processor_override( # Ensure we have the right number of placeholders per num_crops size hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token_id = tokenizer.convert_tokens_to_ids( - getattr(hf_processor, "image_token") # noqa: B009 - ) + image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token) img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id) pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values"] assert isinstance(pixel_values, torch.Tensor) diff --git a/tests/models/multimodal/processing/test_qwen3_omni.py b/tests/models/multimodal/processing/test_qwen3_omni.py index 4a458630f24e..c0ebf0297184 100644 --- a/tests/models/multimodal/processing/test_qwen3_omni.py +++ b/tests/models/multimodal/processing/test_qwen3_omni.py @@ -58,9 +58,7 @@ def test_processor_with_audio_sample_rate( # Verify audio tokens are generated hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - audio_token_id = tokenizer.convert_tokens_to_ids( - getattr(hf_processor, "audio_token") # noqa: B009 - ) + audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token) aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id) assert aud_tok_count >= 1, ( @@ -100,9 +98,7 @@ def get_token_count(duration: float) -> int: hf_processor_mm_kwargs=hf_processor_mm_kwargs, ) hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs) - audio_token_id = tokenizer.convert_tokens_to_ids( - getattr(hf_proc, "audio_token") # noqa: B009 - ) + audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token) return processed["prompt_token_ids"].count(audio_token_id) short_tokens = get_token_count(1.0) diff --git a/tests/models/multimodal/processing/test_smolvlm.py b/tests/models/multimodal/processing/test_smolvlm.py index 4ce67d2cd2a7..f259eb7d4f02 100644 --- a/tests/models/multimodal/processing/test_smolvlm.py +++ b/tests/models/multimodal/processing/test_smolvlm.py @@ -72,9 +72,7 @@ def test_processor_override( processor=hf_processor, mm_kwargs=hf_processor_mm_kwargs, ) - assert num_patches == expected_toks_per_img // getattr( # noqa: B009 - hf_processor, "image_seq_len" - ) + assert num_patches == expected_toks_per_img // hf_processor.image_seq_len hf_processed_inputs = hf_processor( text=prompt, diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index 17dc63971a36..573bdd4a8d8b 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -246,13 +246,11 @@ def test_model_tensor_schema(model_id: str): def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions: if modality == "video": - return VideoDummyOptions( - count=count, num_frames=None, width=None, height=None - ) + return VideoDummyOptions(count=count) if modality == "image": - return ImageDummyOptions(count=count, width=None, height=None) + return ImageDummyOptions(count=count) if modality == "audio": - return AudioDummyOptions(count=count, length=None) + return AudioDummyOptions(count=count) return BaseDummyOptions(count=count) model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions( diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py index 8c567753987e..b0dc9ae0cd57 100644 --- a/tests/models/multimodal/processing/test_transformers_audio.py +++ b/tests/models/multimodal/processing/test_transformers_audio.py @@ -123,7 +123,7 @@ def test_audio_multiple_inputs(separator): ModelConfig(model=model_id, model_impl="transformers") ) - audio_token = getattr(mm_processor.info.get_hf_processor(), "audio_token") # noqa: B009 + audio_token = mm_processor.info.get_hf_processor().audio_token # One token per audio; the processor expands each to its placeholder run. prompt = ( "<|start_of_role|>user<|end_of_role|>" diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py index 61b6720e5967..433a00942b80 100644 --- a/tests/models/multimodal/processing/test_transformers_image.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -99,7 +99,7 @@ def test_batch_padding_removed_from_image_items(): mm_processor = MULTIMODAL_REGISTRY.create_processor( ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers") ) - image_token = getattr(mm_processor.info.get_hf_processor(), "image_token") # noqa: B009 + image_token = mm_processor.info.get_hf_processor().image_token images = [ ImageAsset("cherry_blossom").pil_image, @@ -131,7 +131,7 @@ def _process_one_gemma3_image(): mm_processor = MULTIMODAL_REGISTRY.create_processor( ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers") ) - boi_token = getattr(mm_processor.info.get_hf_processor(), "boi_token") # noqa: B009 + boi_token = mm_processor.info.get_hf_processor().boi_token return mm_processor( prompt=f"{boi_token} What is this?", mm_items=mm_processor.info.parse_mm_data( diff --git a/tests/models/multimodal/processing/test_transformers_video.py b/tests/models/multimodal/processing/test_transformers_video.py index b454e96f179a..00966495f6e0 100644 --- a/tests/models/multimodal/processing/test_transformers_video.py +++ b/tests/models/multimodal/processing/test_transformers_video.py @@ -96,7 +96,7 @@ def test_video_multimodal_processor(model_id): assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist() (placeholder,) = result["mm_placeholders"]["video"] - (num_video_tokens,) = getattr(hf_processor, "_get_num_multimodal_tokens")( # noqa: B009 + (num_video_tokens,) = hf_processor._get_num_multimodal_tokens( video_sizes=[frames.shape[:3]] )["num_video_tokens"] assert placeholder.get_num_embeds() == num_video_tokens @@ -288,7 +288,7 @@ def test_dummy_video_has_the_most_tokens(model_id): assert placeholder.get_num_embeds() == max_tokens # Shrinking the dummy frames must never cost tokens num_frames = info.get_num_frames_with_most_features(seq_len, {"video": 1}) - (huge_frame_tokens,) = getattr( # noqa: B009 - info.get_hf_processor(), "_get_num_multimodal_tokens" - )(video_sizes=[[num_frames, 10_000, 10_000]])["num_video_tokens"] + (huge_frame_tokens,) = info.get_hf_processor()._get_num_multimodal_tokens( + video_sizes=[[num_frames, 10_000, 10_000]] + )["num_video_tokens"] assert max_tokens >= huge_frame_tokens diff --git a/tests/models/multimodal/processing/test_unlimited_ocr.py b/tests/models/multimodal/processing/test_unlimited_ocr.py index 06f68f332cd5..4f5c08504725 100644 --- a/tests/models/multimodal/processing/test_unlimited_ocr.py +++ b/tests/models/multimodal/processing/test_unlimited_ocr.py @@ -31,11 +31,11 @@ def test_profiling_tokens_cover_all_crop_grids(): budget = dummy_inputs["mm_placeholders"]["image"][0].get_num_embeds() for width_tiles, height_tiles in calculate_aspect_ratios( - max_num=getattr(hf_processor, "max_crops") # noqa: B009 + max_num=hf_processor.max_crops ): num_tokens = mm_processor.info.get_num_image_tokens( - image_width=width_tiles * getattr(hf_processor, "image_size"), # noqa: B009 - image_height=height_tiles * getattr(hf_processor, "image_size"), # noqa: B009 + image_width=width_tiles * hf_processor.image_size, + image_height=height_tiles * hf_processor.image_size, ) assert num_tokens <= budget, (width_tiles, height_tiles, num_tokens, budget) diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 5a69787b73af..db3d86909a80 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -212,31 +212,31 @@ def _merge_mm_processor_kwargs( class BaseDummyOptions: """Base options for generating dummy data during profiling.""" - count: int = Field(999, ge=0) + count: int = Field(default=999, ge=0) @dataclass(config=ConfigDict(extra="forbid")) class VideoDummyOptions(BaseDummyOptions): """Options for generating dummy video data during profiling.""" - num_frames: int | None = Field(None, gt=0) - width: int | None = Field(None, gt=0) - height: int | None = Field(None, gt=0) + num_frames: int | None = Field(default=None, gt=0) + width: int | None = Field(default=None, gt=0) + height: int | None = Field(default=None, gt=0) @dataclass(config=ConfigDict(extra="forbid")) class ImageDummyOptions(BaseDummyOptions): """Options for generating dummy image data during profiling.""" - width: int | None = Field(None, gt=0) - height: int | None = Field(None, gt=0) + width: int | None = Field(default=None, gt=0) + height: int | None = Field(default=None, gt=0) @dataclass(config=ConfigDict(extra="forbid")) class AudioDummyOptions(BaseDummyOptions): """Options for generating dummy audio data during profiling.""" - length: int | None = Field(None, gt=0) + length: int | None = Field(default=None, gt=0) @final diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 701c24a072d4..aabb85818fcd 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -736,7 +736,7 @@ class SupportsLoRA(Protocol): packed_modules_mapping: dict[str, list[str]] = {} # Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers) lora_skip_prefixes: ClassVar[list[str]] = [] - lora_manager: "LoRAModelManager | None" + lora_manager: "LoRAModelManager | None" = None # We can't use runtime_checkable with ClassVar for issubclass checks From b3438986d388e63df941790dcbaeb2e58f64e3e9 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:43:45 +0000 Subject: [PATCH 3/6] Restore original granite_speech input and mistral3 uint8 coverage - Pass granite_speech audio as a bare array again; the (audio, sr) tuple changed the HF and vLLM inputs and is not needed for mypy - Check every raw pixel value is uint8 instead of only those reached by zip Co-Authored-By: Claude Opus 5.5 Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- tests/models/multimodal/generation/test_granite_speech.py | 2 +- tests/models/multimodal/processing/test_mistral3.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/models/multimodal/generation/test_granite_speech.py b/tests/models/multimodal/generation/test_granite_speech.py index f5ee95302543..e1fee635ff66 100644 --- a/tests/models/multimodal/generation/test_granite_speech.py +++ b/tests/models/multimodal/generation/test_granite_speech.py @@ -169,7 +169,7 @@ def test_models( hf_runner, vllm_runner, [ - ([HF_AUDIO_PROMPT], [(audio, int(sr))]), + ([HF_AUDIO_PROMPT], [audio]), ], model, dtype=dtype, diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py index 706737756ff3..6efe721c7a0f 100644 --- a/tests/models/multimodal/processing/test_mistral3.py +++ b/tests/models/multimodal/processing/test_mistral3.py @@ -259,6 +259,10 @@ def test_mm_device_do_normalize(): raw_inputs = processor(prompt, mm_items=mm_items) raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] assert isinstance(raw_values, list) + assert all( + isinstance(value, torch.Tensor) and value.dtype == torch.uint8 + for value in raw_values + ) input_norm = build_mm_input_norm(ctx.model_config) patch_size = processor.info.get_hf_config().vision_config.patch_size @@ -277,7 +281,6 @@ def pack_patches(image: torch.Tensor) -> torch.Tensor: for raw, normalized in zip(raw_values, normalized_values): assert isinstance(raw, torch.Tensor) assert isinstance(normalized, torch.Tensor) - assert raw.dtype == torch.uint8 output = input_norm(pack_patches(raw), normalized.dtype) torch.testing.assert_close( output, pack_patches(normalized), rtol=1e-5, atol=1e-6 From bd553a0ef9c86653283c51e7f0372ef878da3e9c Mon Sep 17 00:00:00 2001 From: yewentao256 Date: Thu, 29 Oct 2026 15:30:03 +0000 Subject: [PATCH 4/6] fix Signed-off-by: yewentao256 --- vllm/model_executor/models/ming_v3_asr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/ming_v3_asr.py b/vllm/model_executor/models/ming_v3_asr.py index 29e083407980..384e3c2bb06b 100644 --- a/vllm/model_executor/models/ming_v3_asr.py +++ b/vllm/model_executor/models/ming_v3_asr.py @@ -58,7 +58,7 @@ from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import MultiModalDummyOptions from vllm.config.speech_to_text import SpeechToTextParams -from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt +from vllm.inputs import MultiModalDataDict, TokensPrompt from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser @@ -441,7 +441,7 @@ def get_speech_to_text_config( ) @classmethod - def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> TokensPrompt: if stt_params.task_type != "transcribe" or stt_params.to_language is not None: raise ValueError("Ming V3 ASR does not support translation") language = cls.validate_language(stt_params.language) From e40df1fdd351fef84a060227acc8085bd8581e7a Mon Sep 17 00:00:00 2001 From: yewentao256 Date: Thu, 29 Oct 2026 18:31:19 +0000 Subject: [PATCH 5/6] fix Signed-off-by: yewentao256 --- tests/models/multimodal/processing/test_mistral3.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py index 6efe721c7a0f..6d411b9b249e 100644 --- a/tests/models/multimodal/processing/test_mistral3.py +++ b/tests/models/multimodal/processing/test_mistral3.py @@ -253,18 +253,19 @@ def test_mm_device_do_normalize(): normalized_inputs = processor(prompt, mm_items=mm_items) normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] - assert isinstance(normalized_values, list) + assert isinstance(normalized_values, (list, torch.Tensor)) ctx.model_config.multimodal_config.mm_device_do_normalize = True raw_inputs = processor(prompt, mm_items=mm_items) raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] - assert isinstance(raw_values, list) + assert isinstance(raw_values, (list, torch.Tensor)) assert all( isinstance(value, torch.Tensor) and value.dtype == torch.uint8 for value in raw_values ) - input_norm = build_mm_input_norm(ctx.model_config) + device = torch.device(current_platform.device_type) + input_norm = build_mm_input_norm(ctx.model_config).to(device) patch_size = processor.info.get_hf_config().vision_config.patch_size def pack_patches(image: torch.Tensor) -> torch.Tensor: @@ -281,9 +282,9 @@ def pack_patches(image: torch.Tensor) -> torch.Tensor: for raw, normalized in zip(raw_values, normalized_values): assert isinstance(raw, torch.Tensor) assert isinstance(normalized, torch.Tensor) - output = input_norm(pack_patches(raw), normalized.dtype) + output = input_norm(pack_patches(raw).to(device), normalized.dtype) torch.testing.assert_close( - output, pack_patches(normalized), rtol=1e-5, atol=1e-6 + output, pack_patches(normalized).to(device), rtol=1e-5, atol=1e-6 ) From 39fb022a069859765bade476244cf90618fd33d8 Mon Sep 17 00:00:00 2001 From: yewentao256 Date: Thu, 29 Oct 2026 23:25:32 +0000 Subject: [PATCH 6/6] fix precommit Signed-off-by: yewentao256 --- tests/models/multimodal/test_nano_nemotron_vl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/models/multimodal/test_nano_nemotron_vl.py b/tests/models/multimodal/test_nano_nemotron_vl.py index 7c880d8897d1..ee6f36893f9e 100644 --- a/tests/models/multimodal/test_nano_nemotron_vl.py +++ b/tests/models/multimodal/test_nano_nemotron_vl.py @@ -60,7 +60,7 @@ def test_nemotron_resized_normalization_matches_reference( if not do_cpu_normalize: assert output.dtype == torch.uint8 - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() nn.Module.__init__(model) model.input_norm = FusedMMInputNorm( mean.flatten().tolist(), std.flatten().tolist(), 1.0 / 255.0 @@ -89,7 +89,7 @@ def test_nemotron_dynamic_device_normalization_matches_cpu_reference( shape = (1, 7, 3 * 16 * 16) pixels = torch.randint(0, 256, shape, dtype=torch.uint8) - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() nn.Module.__init__(model) model.input_norm = FusedMMInputNorm(image_mean, image_std, 1.0 / 255.0).to(_DEVICE) model.llm_dtype = torch.bfloat16 @@ -107,7 +107,7 @@ def test_nemotron_dynamic_device_normalization_matches_cpu_reference( def test_nemotron_cpu_normalized_pixels_bypass_input_norm(): pixels = torch.randn(1, 3, 16, 16) - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() nn.Module.__init__(model) model.input_norm = None @@ -175,7 +175,7 @@ def test_nemotron_processor_defers_normalization_to_device( # Normalizing on the device hands the encoder the same dtype as the CPU # path. Values differ only where the uint8 resize rounds or clips the # bicubic overshoot: a few elements by several levels, tiny on average. - model = object.__new__(NemotronH_Nano_VL_V2) + model = _uninitialized_model() nn.Module.__init__(model) model.input_norm = FusedMMInputNorm( config.norm_mean, config.norm_std, 1.0 / 255.0