diff --git a/tests/models/multimodal/conftest.py b/tests/models/multimodal/conftest.py
index 54e545d6bf04..c00932738f47 100644
--- a/tests/models/multimodal/conftest.py
+++ b/tests/models/multimodal/conftest.py
@@ -10,4 +10,4 @@
# longer re-exports the typing aliases, so executing that module raises NameError.
# TODO: Remove once the checkpoint fixes its imports.
if not hasattr(F, "List"):
- F.List = list
+ F.List = list # type: ignore[attr-defined]
diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py
index 9ea2cc43cd6e..083eba8b5639 100644
--- a/tests/models/multimodal/generation/test_common.py
+++ b/tests/models/multimodal/generation/test_common.py
@@ -697,7 +697,7 @@ def _granite4_vision_vllm_to_hf_output(vllm_output, model):
img_idx_to_prompt=lambda idx: "(./)\n",
max_model_len=4096,
max_num_seqs=2,
- get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id],
+ get_stop_token_ids=lambda tok: [getattr(tok, "eos_id"), getattr(tok, "eot_id")], # noqa: B009
hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
patch_hf_runner=model_utils.minicpmv_25_patch_hf_runner,
),
@@ -927,7 +927,7 @@ def _granite4_vision_vllm_to_hf_output(vllm_output, model):
auto_cls=AutoModelForImageTextToText,
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
marks=multi_gpu_marks(num_gpus=2),
- **COMMON_BROADCAST_SETTINGS, # type: ignore
+ **COMMON_BROADCAST_SETTINGS,
),
"llava_next-broadcast": VLMTestInfo(
models=["llava-hf/llava-v1.6-mistral-7b-hf"],
@@ -936,7 +936,7 @@ def _granite4_vision_vllm_to_hf_output(vllm_output, model):
auto_cls=AutoModelForImageTextToText,
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
marks=multi_gpu_marks(num_gpus=2),
- **COMMON_BROADCAST_SETTINGS, # type: ignore
+ **COMMON_BROADCAST_SETTINGS,
),
### Custom input edge-cases for specific models
"intern_vl-diff-patches": VLMTestInfo(
diff --git a/tests/models/multimodal/generation/test_maverick.py b/tests/models/multimodal/generation/test_maverick.py
index 0f6bec107fc2..a44491dfec31 100644
--- a/tests/models/multimodal/generation/test_maverick.py
+++ b/tests/models/multimodal/generation/test_maverick.py
@@ -523,7 +523,7 @@ def save_weights_to_safetensors(
print(f"Saved shard {i + 1}/{len(shards)}: {filename}")
# Create index file
- index_data = {
+ index_data: dict[str, Any] = {
"metadata": {
"total_size": sum(
tensor.numel() * tensor.element_size() for tensor in weights.values()
diff --git a/tests/models/multimodal/generation/test_nemotron_parse_35.py b/tests/models/multimodal/generation/test_nemotron_parse_35.py
index bb8fbf51d937..f5b9884df4c7 100644
--- a/tests/models/multimodal/generation/test_nemotron_parse_35.py
+++ b/tests/models/multimodal/generation/test_nemotron_parse_35.py
@@ -51,7 +51,9 @@ def generate_hf_logprobs_from_logits(
}
)
token_ids = generated.sequences[0, -len(logprobs) :].tolist()
- outputs.append((token_ids, hf_model.tokenizer.decode(token_ids), logprobs))
+ decoded = hf_model.tokenizer.decode(token_ids)
+ assert isinstance(decoded, str)
+ outputs.append((token_ids, decoded, logprobs))
return outputs
diff --git a/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py b/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py
index 918bc51ea657..55d5108ee09b 100644
--- a/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py
+++ b/tests/models/multimodal/generation/test_nemotron_parse_35_mtp.py
@@ -23,10 +23,11 @@
def test_parse_mtp_config_routes_only_causal_prefix_architecture() -> None:
- config = PreTrainedConfig(
- architectures=["RadioMBartPrefixForCausalLM"],
- decoder=PreTrainedConfig(causal_vision_prefix=True, vocab_size=8, d_model=4),
- )
+ config = PreTrainedConfig(architectures=["RadioMBartPrefixForCausalLM"])
+ config.decoder = PreTrainedConfig()
+ config.decoder.causal_vision_prefix = True
+ config.decoder.vocab_size = 8
+ config.decoder.d_model = 4
draft = SpeculativeConfig.hf_config_override(config)
diff --git a/tests/models/multimodal/generation/test_qwen2_vl.py b/tests/models/multimodal/generation/test_qwen2_vl.py
index 4508135b2bb9..ea74d117b302 100644
--- a/tests/models/multimodal/generation/test_qwen2_vl.py
+++ b/tests/models/multimodal/generation/test_qwen2_vl.py
@@ -126,6 +126,7 @@ def batch_make_image_embeddings(
image_processor = processor.image_processor
mm_config = llm.get_llm().model_config.multimodal_config
+ assert mm_config is not None
mm_kwargs = mm_config.merge_mm_processor_kwargs({})
preprocess_result = image_processor.preprocess(
images=images, return_tensors="pt", **mm_kwargs
@@ -211,6 +212,7 @@ def batch_make_video_embeddings(
video_processor = processor.video_processor
mm_config = llm.get_llm().model_config.multimodal_config
+ assert mm_config is not None
mm_kwargs = mm_config.merge_mm_processor_kwargs({})
preprocess_result = video_processor.preprocess(
videos=videos, return_tensors="pt", **mm_kwargs
diff --git a/tests/models/multimodal/generation/test_ultravox.py b/tests/models/multimodal/generation/test_ultravox.py
index 72f473835800..8ae5e573b4fb 100644
--- a/tests/models/multimodal/generation/test_ultravox.py
+++ b/tests/models/multimodal/generation/test_ultravox.py
@@ -24,7 +24,7 @@
MULTI_AUDIO_PROMPT = "Describe each of the audios above."
-AudioTuple = tuple[np.ndarray, int]
+AudioTuple = tuple[np.ndarray, float]
VLLM_PLACEHOLDER = "<|audio|>"
HF_PLACEHOLDER = "<|audio|>"
@@ -92,7 +92,7 @@ def _get_prompt(audio_count, question, placeholder):
def run_multi_audio_test(
vllm_runner: type[VllmRunner],
- prompts_and_audios: list[tuple[str, list[AudioTuple]]],
+ prompts_and_audios: list[tuple[str | list[int], list[AudioTuple]]],
model: str,
*,
dtype: str,
diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py
index e80dd9dc8515..8751a94fe690 100644
--- a/tests/models/multimodal/generation/test_vit_cudagraph.py
+++ b/tests/models/multimodal/generation/test_vit_cudagraph.py
@@ -34,7 +34,7 @@ class VitCudagraphTestConfig:
def params_with_marks(
configs: dict[str, VitCudagraphTestConfig],
-) -> list[pytest.param]:
+) -> list[object]:
return [
pytest.param(model_id, marks=cfg.marks) for model_id, cfg in configs.items()
]
@@ -520,6 +520,7 @@ def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
"baby_reading": config.video_prompt, # type: ignore[typeddict-item]
}
)
+ sampled_vids: list
if config.needs_video_metadata:
sampled_vids = [
sample_frames_with_video_metadata(
diff --git a/tests/models/multimodal/generation/test_voxtral.py b/tests/models/multimodal/generation/test_voxtral.py
index 3c9d2ace8cca..434df130a080 100644
--- a/tests/models/multimodal/generation/test_voxtral.py
+++ b/tests/models/multimodal/generation/test_voxtral.py
@@ -58,7 +58,7 @@ def test_models_with_multiple_audios(
vllm_prompt = _get_prompt(audio_assets, MULTI_AUDIO_PROMPT)
run_multi_audio_test(
vllm_runner,
- [(vllm_prompt, [a.audio_and_sample_rate for a in audio_assets])], # type: ignore[list-item]
+ [(vllm_prompt, [a.audio_and_sample_rate for a in audio_assets])],
MODEL_NAME,
dtype=dtype,
max_tokens=max_tokens,
diff --git a/tests/models/multimodal/generation/vlm_utils/builders.py b/tests/models/multimodal/generation/vlm_utils/builders.py
index 1b7e2347be2f..e73c69c13897 100644
--- a/tests/models/multimodal/generation/vlm_utils/builders.py
+++ b/tests/models/multimodal/generation/vlm_utils/builders.py
@@ -224,7 +224,7 @@ def build_embedding_inputs_from_test_info(
images = [asset.pil_image for asset in image_assets]
embeds = test_info.convert_assets_to_embeddings(image_assets)
if test_info.dtype != "auto":
- dtype = getattr(torch, test_info.dtype) # type: ignore
+ dtype = getattr(torch, test_info.dtype)
embeds = [e.to(dtype=dtype) for e in embeds]
assert len(images) == len(model_prompts)
diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py
index 9acbe2c0d6b3..f201d5cc2137 100644
--- a/tests/models/multimodal/generation/vlm_utils/model_utils.py
+++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py
@@ -22,6 +22,7 @@
GenerationMixin,
)
from transformers.masking_utils import create_causal_mask
+from transformers.utils import ModelOutput
from transformers.video_utils import VideoMetadata
from vllm.logprobs import SampleLogprobs
@@ -105,7 +106,9 @@ def _llava_vllm_to_hf_output(
hf_output_str = output_str[1:] if output_str[0] == " " else output_str
if hf_output_ids[-1] == eos_token_id:
- hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
+ decoded_eos = tokenizer.decode(eos_token_id)
+ assert isinstance(decoded_eos, str)
+ hf_output_str = hf_output_str + decoded_eos
return hf_output_ids, hf_output_str, out_logprobs
@@ -137,7 +140,9 @@ def llava_onevision_vllm_to_hf_output(
hf_output_str = output_str
if hf_output_ids[-1] == eos_token_id:
- hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
+ decoded_eos = tokenizer.decode(eos_token_id)
+ assert isinstance(decoded_eos, str)
+ hf_output_str = hf_output_str + decoded_eos
return hf_output_ids, hf_output_str, out_logprobs
@@ -179,7 +184,9 @@ def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> Runner
hf_output_str = output_str
if hf_output_ids[-1] == eos_token_id:
- hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
+ decoded_eos = tokenizer.decode(eos_token_id)
+ assert isinstance(decoded_eos, str)
+ hf_output_str = hf_output_str + decoded_eos
return hf_output_ids, hf_output_str, out_logprobs
@@ -286,7 +293,9 @@ def gemma3_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOut
hf_output_str = output_str
if hf_output_ids[-1] == eos_token_id:
- hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
+ decoded_eos = tokenizer.decode(eos_token_id)
+ assert isinstance(decoded_eos, str)
+ hf_output_str = hf_output_str + decoded_eos
return hf_output_ids, hf_output_str, out_logprobs
@@ -572,11 +581,10 @@ def patched_forward(
# Prepare attention mask
attention_mask = create_causal_mask(
config=self.config,
- input_embeds=inputs_embeds,
+ inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
past_key_values=past_key_values,
position_ids=position_ids,
- cache_position=cache_position,
)
# Initialize and collect hidden states
@@ -800,7 +808,7 @@ def _internvl_generate(
input_ids: torch.FloatTensor,
attention_mask: torch.LongTensor | None = None,
**generate_kwargs,
-) -> torch.LongTensor:
+) -> torch.Tensor | ModelOutput:
"""Generate method for InternVL2 model without fixed use_cache."""
assert self.img_context_token_id is not None
target_dtype = next(self.parameters()).dtype
@@ -1207,7 +1215,7 @@ def patched_generate(*args, **kwargs):
return output
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign]
- hf_model.model.generate = patched_generate # type: ignore[method-assign]
+ hf_model.model.generate = patched_generate
return hf_model
diff --git a/tests/models/multimodal/generation_ppl_test/ppl_utils.py b/tests/models/multimodal/generation_ppl_test/ppl_utils.py
index c612eb7141da..d19764740995 100644
--- a/tests/models/multimodal/generation_ppl_test/ppl_utils.py
+++ b/tests/models/multimodal/generation_ppl_test/ppl_utils.py
@@ -17,7 +17,7 @@
PPL_TOL = 0.01
-@torch.inference_mode
+@torch.inference_mode()
def vqa_ppl_test(
hf_runner,
vllm_runner,
diff --git a/tests/models/multimodal/pooling/test_dse_qwen2_vl.py b/tests/models/multimodal/pooling/test_dse_qwen2_vl.py
index 0fff8f40ac74..8b975c0ada48 100644
--- a/tests/models/multimodal/pooling/test_dse_qwen2_vl.py
+++ b/tests/models/multimodal/pooling/test_dse_qwen2_vl.py
@@ -180,7 +180,7 @@ def test_models_text(
hf_runner,
vllm_runner,
input_texts,
- input_images, # type: ignore
+ input_images,
embed_texts,
model,
dtype=dtype,
diff --git a/tests/models/multimodal/pooling/test_embedding_gemma2.py b/tests/models/multimodal/pooling/test_embedding_gemma2.py
index 0dcb72406da5..ff4f7e9fb167 100644
--- a/tests/models/multimodal/pooling/test_embedding_gemma2.py
+++ b/tests/models/multimodal/pooling/test_embedding_gemma2.py
@@ -9,6 +9,8 @@
import pytest
import torch
+from vllm.multimodal.video import EmbeddingGemma2VideoBackend
+
MODEL = os.environ.get("EG2_MODEL_PATH")
needs_model = pytest.mark.skipif(not MODEL, reason="EG2_MODEL_PATH not set")
TEXT_COS, MM_COS = 0.999, 0.999
@@ -348,6 +350,7 @@ def test_video_loader_real_video(vllm_runner, st_model):
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
+ assert issubclass(backend_cls, EmbeddingGemma2VideoBackend)
with open(video_path, "rb") as f:
data = f.read()
@@ -396,6 +399,7 @@ def test_video_loader_duplicate_expansion_e2e(vllm_runner, st_model):
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
+ assert issubclass(backend_cls, EmbeddingGemma2VideoBackend)
# 4s clip at 0.5 fps = 2 unique frames
lowfps_path = "/tmp/test_lowfps_05.mp4"
_ensure_synthetic_video(lowfps_path, duration=4.0, fps=0.5, width=64, height=48)
@@ -436,6 +440,7 @@ def test_video_loader_indices():
)
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
+ assert issubclass(backend_cls, EmbeddingGemma2VideoBackend)
hf_proc = EmbeddingGemma2VideoProcessor()
# Case 1: clip shorter than 32s (e.g. 5s @ 24fps = 120 frames)
@@ -521,6 +526,7 @@ def test_video_loader_duplicate_expansion():
)
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
+ assert issubclass(backend_cls, EmbeddingGemma2VideoBackend)
hf_proc = EmbeddingGemma2VideoProcessor()
# 4s clip @ 0.5fps = 2 frames total. Requested target fps = 1.0 -> 4 frames.
diff --git a/tests/models/multimodal/pooling/test_intern_vit.py b/tests/models/multimodal/pooling/test_intern_vit.py
index 4d5e8728589c..7b5db69c6b9a 100644
--- a/tests/models/multimodal/pooling/test_intern_vit.py
+++ b/tests/models/multimodal/pooling/test_intern_vit.py
@@ -49,14 +49,14 @@ def _patched_init(self, *args, **kwargs):
_orig_init(self, *args, **kwargs)
self.all_tied_weights_keys = {}
- PreTrainedModel.__init__ = _patched_init
+ PreTrainedModel.__init__ = _patched_init # type: ignore[method-assign]
try:
hf_model = AutoModel.from_pretrained(
model, dtype=torch_dtype, trust_remote_code=True
).to(DEVICE_TYPE)
finally:
- PreTrainedModel.__init__ = _orig_init
+ PreTrainedModel.__init__ = _orig_init # type: ignore[method-assign]
hf_outputs_per_image = [
hf_model(pixel_value.to(DEVICE_TYPE)).last_hidden_state
for pixel_value in pixel_values
diff --git a/tests/models/multimodal/pooling/test_llava_next.py b/tests/models/multimodal/pooling/test_llava_next.py
index 97bdaea4ecbe..a329ce4fece3 100644
--- a/tests/models/multimodal/pooling/test_llava_next.py
+++ b/tests/models/multimodal/pooling/test_llava_next.py
@@ -126,7 +126,7 @@ def test_models_text(
hf_runner,
vllm_runner,
input_texts,
- input_images, # type: ignore
+ input_images,
model,
dtype=dtype,
)
diff --git a/tests/models/multimodal/pooling/test_phi3v.py b/tests/models/multimodal/pooling/test_phi3v.py
index ba017f065a13..e7442177c3d0 100644
--- a/tests/models/multimodal/pooling/test_phi3v.py
+++ b/tests/models/multimodal/pooling/test_phi3v.py
@@ -119,7 +119,7 @@ def test_models_text(
hf_runner,
vllm_runner,
input_texts,
- input_images, # type: ignore
+ input_images,
model,
dtype=dtype,
)
diff --git a/tests/models/multimodal/processing/test_audio_in_video.py b/tests/models/multimodal/processing/test_audio_in_video.py
index 6b77e8834e12..a8e164d7f39d 100644
--- a/tests/models/multimodal/processing/test_audio_in_video.py
+++ b/tests/models/multimodal/processing/test_audio_in_video.py
@@ -178,9 +178,11 @@ def test_audio_in_video_metadata_only(
mm_config.enable_mm_embeds = True
mm_config.allow_missing_mm_embeddings = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
- prompt = "".join(
+ video_placeholders = [
model_cls.get_placeholder_str("video", i) for i in range(num_videos)
- )
+ ]
+ assert all(isinstance(token, str) for token in video_placeholders)
+ prompt = "".join(token for token in video_placeholders if token is not None)
kwargs = {"use_audio_in_video": True}
uuids = {
modality: [f"source-{i}" for i in range(num_videos)]
@@ -192,7 +194,9 @@ def test_audio_in_video_metadata_only(
mm_data["image"] = [Image.new("RGB", (64, 64), "green")]
uuids["image"] = ["image-source"]
- prompt = model_cls.get_placeholder_str("image", 0) + prompt
+ image_placeholder = model_cls.get_placeholder_str("image", 0)
+ assert image_placeholder is not None
+ prompt = image_placeholder + prompt
raw = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
@@ -237,9 +241,16 @@ def test_audio_in_video_metadata_only(
"uuid": "image-source",
},
)
- parse = parse_chat_messages_async if is_async else parse_chat_messages
- parsed = parse(messages, ctx.model_config, "openai", mm_processor_kwargs=kwargs)
- conversation, metadata, remote_uuids = asyncio.run(parsed) if is_async else parsed
+ if is_async:
+ conversation, metadata, remote_uuids = asyncio.run(
+ parse_chat_messages_async(
+ messages, ctx.model_config, "openai", mm_processor_kwargs=kwargs
+ )
+ )
+ else:
+ conversation, metadata, remote_uuids = parse_chat_messages(
+ messages, ctx.model_config, "openai", mm_processor_kwargs=kwargs
+ )
assert conversation[0]["content"] == (
([{"type": "image"}] if has_image else []) + [{"type": "video"}] * num_videos
)
@@ -271,7 +282,7 @@ def unexpected_preprocessing(*args, **kwargs):
)
model = model_cls.__new__(model_cls)
- model.config = processor.info.get_hf_config()
+ setattr(model, "config", processor.info.get_hf_config()) # noqa: B010
remote_features = [
MultiModalFeatureSpec(
data=remote["mm_kwargs"][f.modality][int(f.identifier.split("-")[-1])],
@@ -281,10 +292,9 @@ def unexpected_preprocessing(*args, **kwargs):
)
for f in features
]
- raw_positions, raw_delta = model.get_mrope_input_positions(
- raw["prompt_token_ids"], features
- )
- remote_positions, remote_delta = model.get_mrope_input_positions(
+ get_positions = getattr(model, "get_mrope_input_positions") # noqa: B009
+ raw_positions, raw_delta = get_positions(raw["prompt_token_ids"], features)
+ remote_positions, remote_delta = get_positions(
remote["prompt_token_ids"], remote_features
)
assert raw_delta == remote_delta
diff --git a/tests/models/multimodal/processing/test_audioflamingo3.py b/tests/models/multimodal/processing/test_audioflamingo3.py
index c3adccb27d59..16182eac3052 100644
--- a/tests/models/multimodal/processing/test_audioflamingo3.py
+++ b/tests/models/multimodal/processing/test_audioflamingo3.py
@@ -134,7 +134,9 @@ def test_dummy_data_generation(mock_ctx):
assert len(dummy_data["audio"]) == 2
expected_len = 60 * 16000
- assert len(dummy_data["audio"][0]) == expected_len
+ audio_data = dummy_data["audio"]
+ assert isinstance(audio_data, list)
+ assert len(audio_data[0]) == expected_len
def test_audio_token_count_matches_hf_processor_math():
diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py
index 667907c3e55c..7216c47bd347 100644
--- a/tests/models/multimodal/processing/test_common.py
+++ b/tests/models/multimodal/processing/test_common.py
@@ -43,6 +43,7 @@
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""Add metadata to video mm_data."""
+ assert isinstance(mm_data, dict)
def create_metadata(frames: np.ndarray):
num_frames = len(frames)
@@ -71,6 +72,7 @@ def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""Patch the multimodal data for GLM-ASR model.
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
"""
+ assert isinstance(mm_data, dict)
if "audio" in mm_data:
audio = mm_data["audio"]
if isinstance(audio, list) and len(audio) > 1:
@@ -218,7 +220,7 @@ def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch):
mm_config.enable_mm_embeds = True
mm_config.allow_missing_mm_embeddings = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
- sample_rate = processor.info.get_feature_extractor().sampling_rate
+ sample_rate = getattr(processor.info, "get_feature_extractor")().sampling_rate # noqa: B009
audios = [
np.zeros(int(duration * sample_rate), dtype=np.float32)
for duration in durations
@@ -333,6 +335,7 @@ def _test_processing_correctness(
)
# Ensure that the cache can fit all of the data
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
+ assert model_config.multimodal_config is not None
model_config.multimodal_config.mm_processor_cache_gb = 2048
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
diff --git a/tests/models/multimodal/processing/test_gemma3.py b/tests/models/multimodal/processing/test_gemma3.py
index 2b4c213695ee..8b61deb91d6f 100644
--- a/tests/models/multimodal/processing/test_gemma3.py
+++ b/tests/models/multimodal/processing/test_gemma3.py
@@ -4,6 +4,7 @@
import pytest
import torch
+from vllm.model_executor.models.gemma3_mm import Gemma3ProcessingInfo
from vllm.model_executor.models.gemma3n_audio_utils import (
adjust_audio_features_to_expected_length,
)
@@ -162,6 +163,7 @@ def test_get_image_size_with_most_features(
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Gemma3ProcessingInfo)
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
@@ -185,5 +187,6 @@ def test_get_image_size_with_most_features(
)
mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
num_patches_tensor = mm_kwargs_data["num_patches"]
+ assert isinstance(num_patches_tensor, torch.Tensor)
tokens = int(num_patches_tensor.item()) * image_seq_length
assert tokens <= max_tokens
diff --git a/tests/models/multimodal/processing/test_gemma4.py b/tests/models/multimodal/processing/test_gemma4.py
index f9d935991e90..90f5d0e24eba 100644
--- a/tests/models/multimodal/processing/test_gemma4.py
+++ b/tests/models/multimodal/processing/test_gemma4.py
@@ -14,6 +14,7 @@
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration,
Gemma4ImagePixelInputs,
+ Gemma4ProcessingInfo,
)
from vllm.model_executor.models.interfaces import supports_multimodal
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -98,6 +99,7 @@ def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Gemma4ProcessingInfo)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
@@ -190,6 +192,7 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Gemma4ProcessingInfo)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
@@ -397,7 +400,7 @@ def test_encoder_chunk_no_free_memory_falls_back_to_one():
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
- model = object.__new__(Gemma4ForConditionalGeneration)
+ model = object.__new__(Gemma4ForConditionalGeneration) # type: ignore[type-abstract]
vision_config = SimpleNamespace(
patch_size=16, pooling_kernel_size=3, default_output_length=280
)
diff --git a/tests/models/multimodal/processing/test_gemma4_unified.py b/tests/models/multimodal/processing/test_gemma4_unified.py
index 28c1018c6602..3754e4abc14f 100644
--- a/tests/models/multimodal/processing/test_gemma4_unified.py
+++ b/tests/models/multimodal/processing/test_gemma4_unified.py
@@ -15,6 +15,7 @@
)
from vllm.model_executor.models.gemma4_unified import (
Gemma4UnifiedForConditionalGeneration,
+ Gemma4UnifiedProcessingInfo,
Gemma4UnifiedVisionEmbedder,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -81,6 +82,7 @@ def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Gemma4UnifiedProcessingInfo)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
@@ -171,6 +173,7 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Gemma4UnifiedProcessingInfo)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
@@ -315,7 +318,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.embedding_projection(x)
-class _MockUnifiedModel:
+class _MockUnifiedModel(Gemma4UnifiedForConditionalGeneration):
+ # Initialize only the modules needed by the borrowed vision methods.
def __init__(
self,
embed_dim: int = 48,
@@ -323,6 +327,7 @@ def __init__(
dtype: torch.dtype = torch.float32,
device: str = "cpu",
):
+ torch.nn.Module.__init__(self)
self.vision_embedder = _build_test_embedder(
embed_dim=embed_dim, max_pos=max_pos, dtype=dtype, device=device
)
@@ -330,11 +335,6 @@ def __init__(
embed_dim=embed_dim, dtype=dtype, device=device
)
- _vision_chunk = staticmethod(Gemma4UnifiedForConditionalGeneration._vision_chunk)
- _embed_vision_patches = Gemma4UnifiedForConditionalGeneration._embed_vision_patches
- _process_image_input = Gemma4UnifiedForConditionalGeneration._process_image_input
- _process_video_input = Gemma4UnifiedForConditionalGeneration._process_video_input
-
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
@pytest.mark.parametrize("shape", [(16, 2), (2, 8, 2), (1, 1, 2)])
diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py
index 111fb982cf7d..e9f3a48ec425 100644
--- a/tests/models/multimodal/processing/test_glm5next.py
+++ b/tests/models/multimodal/processing/test_glm5next.py
@@ -177,10 +177,12 @@ def test_mm_device_do_normalize():
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
+ assert isinstance(normalized_values, torch.Tensor)
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
+ assert isinstance(raw_values, torch.Tensor)
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
diff --git a/tests/models/multimodal/processing/test_h2ovl.py b/tests/models/multimodal/processing/test_h2ovl.py
index 5a54484e1bdd..bc43130d04f1 100644
--- a/tests/models/multimodal/processing/test_h2ovl.py
+++ b/tests/models/multimodal/processing/test_h2ovl.py
@@ -120,7 +120,9 @@ def _run_check(
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
- pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
+ pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"]
+ assert isinstance(pixel_values, torch.Tensor)
+ pixel_shape = pixel_values.shape
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
@@ -204,6 +206,7 @@ def test_image_cache_identity_across_image_counts(
)
processor = MULTIMODAL_REGISTRY.create_processor(config)
tokenizer = processor.info.get_tokenizer()
+ images: list[torch.Tensor] | list[Image.Image]
if use_embeds:
images = [torch.ones(8, config.get_hidden_size())] * 3
else:
diff --git a/tests/models/multimodal/processing/test_internvl.py b/tests/models/multimodal/processing/test_internvl.py
index ab5fbe9a0130..de126c2c95fb 100644
--- a/tests/models/multimodal/processing/test_internvl.py
+++ b/tests/models/multimodal/processing/test_internvl.py
@@ -5,6 +5,7 @@
from collections.abc import Mapping
import pytest
+import torch
from PIL import Image
from transformers import PreTrainedConfig
@@ -75,7 +76,9 @@ def _run_check(
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
- pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
+ pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"]
+ assert isinstance(pixel_values, torch.Tensor)
+ pixel_shape = pixel_values.shape
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
diff --git a/tests/models/multimodal/processing/test_kimi_k3.py b/tests/models/multimodal/processing/test_kimi_k3.py
index 62c99b24f143..599b0413e0bc 100644
--- a/tests/models/multimodal/processing/test_kimi_k3.py
+++ b/tests/models/multimodal/processing/test_kimi_k3.py
@@ -23,7 +23,7 @@
def image_processors():
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
- k25_fused = processor.info.image_processor
+ k25_fused = getattr(processor.info, "image_processor") # noqa: B009
assert isinstance(k25_fused, KimiK25FusedVisionProcessor)
k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True)
return k25_fused, k3_hf
@@ -68,10 +68,12 @@ def test_mm_device_do_normalize():
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
+ assert isinstance(normalized_values, torch.Tensor)
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
+ assert isinstance(raw_values, torch.Tensor)
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
diff --git a/tests/models/multimodal/processing/test_lightonocr.py b/tests/models/multimodal/processing/test_lightonocr.py
index 3966bf8cb4dd..c138808c7736 100644
--- a/tests/models/multimodal/processing/test_lightonocr.py
+++ b/tests/models/multimodal/processing/test_lightonocr.py
@@ -47,4 +47,5 @@ def test_lightonocr_image_embeddings_preserve_lengths_and_values(
)
assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes
for item, expected in zip(processed["mm_kwargs"]["image"], embeddings):
+ assert item is not None
torch.testing.assert_close(item.get_data()["image_embeds"], expected)
diff --git a/tests/models/multimodal/processing/test_llama4.py b/tests/models/multimodal/processing/test_llama4.py
index 4bc2e5909980..e3d4a94796b5 100644
--- a/tests/models/multimodal/processing/test_llama4.py
+++ b/tests/models/multimodal/processing/test_llama4.py
@@ -3,7 +3,9 @@
"""Tests for Llama4's multimodal preprocessing kwargs."""
import pytest
+import torch
+from vllm.model_executor.models.mllama4 import Mllama4ProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
@@ -31,6 +33,7 @@ def test_processor_override(
mm_processor_cache_gb=mm_processor_cache_gb,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Mllama4ProcessingInfo)
config = processor.info.get_hf_config()
tokenizer = processor.info.get_tokenizer()
hf_processor = processor.info.get_hf_processor()
@@ -62,6 +65,7 @@ def test_processor_override(
assert prompt_token_ids.count(config.eoi_token_index) == num_imgs
assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs
aspect_ratios = mm_data["aspect_ratios"]
+ assert isinstance(aspect_ratios, torch.Tensor)
num_x_separators = num_y_separators = 0
for tiles_y, tiles_x in aspect_ratios:
if tiles_x * tiles_y > 1:
diff --git a/tests/models/multimodal/processing/test_minicpmv.py b/tests/models/multimodal/processing/test_minicpmv.py
index 59cb4331f3b0..fbcdd77e3f96 100644
--- a/tests/models/multimodal/processing/test_minicpmv.py
+++ b/tests/models/multimodal/processing/test_minicpmv.py
@@ -6,6 +6,7 @@
import numpy as np
import pytest
+from vllm.model_executor.models.minicpmv import MiniCPMVProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import ImageSize
@@ -21,6 +22,7 @@ def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
info = processor.info
+ assert isinstance(info, MiniCPMVProcessingInfo)
processor_1 = info.get_hf_processor(max_slice_nums=1)
processor_2 = info.get_hf_processor(max_slice_nums=2)
assert processor_1.image_processor.max_slice_nums == 1
@@ -42,6 +44,7 @@ def test_image_processor_for_different_models(model_ids):
limit_mm_per_prompt={"image": 1},
)
first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
+ assert isinstance(first_processor.info, MiniCPMVProcessingInfo)
first_image_processor = first_processor.info.get_image_processor()
second_ctx = build_model_context(
@@ -49,6 +52,7 @@ def test_image_processor_for_different_models(model_ids):
limit_mm_per_prompt={"image": 1},
)
second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
+ assert isinstance(second_processor.info, MiniCPMVProcessingInfo)
second_image_processor = second_processor.info.get_image_processor()
second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py
index 87ca8c842bf1..6d411b9b249e 100644
--- a/tests/models/multimodal/processing/test_mistral3.py
+++ b/tests/models/multimodal/processing/test_mistral3.py
@@ -6,7 +6,7 @@
import torch
import torch.nn.functional as F
from PIL import Image
-from transformers import AutoProcessor, BatchFeature
+from transformers import AutoProcessor, BatchFeature, PreTrainedConfig
from transformers.models.pixtral import PixtralProcessor
from vllm.model_executor.layers.fusion.mm_input_norm import (
@@ -79,6 +79,9 @@ def test_pixtral_patch_embed_preserves_image_order_and_strides(
reference = []
for image in images:
if normalize:
+ assert isinstance(norm, FusedMMInputNorm)
+ assert isinstance(norm.weight, torch.Tensor)
+ assert isinstance(norm.bias, torch.Tensor)
normalized = (
image.float() * norm.weight[:, None, None] + norm.bias[:, None, None]
)
@@ -250,13 +253,19 @@ def test_mm_device_do_normalize():
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
+ assert isinstance(normalized_values, (list, torch.Tensor))
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
- assert all(value.dtype == torch.uint8 for value in raw_values)
+ assert isinstance(raw_values, (list, torch.Tensor))
+ assert all(
+ isinstance(value, torch.Tensor) and value.dtype == torch.uint8
+ for value in raw_values
+ )
- input_norm = build_mm_input_norm(ctx.model_config)
+ device = torch.device(current_platform.device_type)
+ input_norm = build_mm_input_norm(ctx.model_config).to(device)
patch_size = processor.info.get_hf_config().vision_config.patch_size
def pack_patches(image: torch.Tensor) -> torch.Tensor:
@@ -271,9 +280,11 @@ def pack_patches(image: torch.Tensor) -> torch.Tensor:
)
for raw, normalized in zip(raw_values, normalized_values):
- output = input_norm(pack_patches(raw), normalized.dtype)
+ assert isinstance(raw, torch.Tensor)
+ assert isinstance(normalized, torch.Tensor)
+ output = input_norm(pack_patches(raw).to(device), normalized.dtype)
torch.testing.assert_close(
- output, pack_patches(normalized), rtol=1e-5, atol=1e-6
+ output, pack_patches(normalized).to(device), rtol=1e-5, atol=1e-6
)
@@ -325,6 +336,6 @@ def test_lightonocr_keeps_vision_config_image_size():
encoder_info = processor.info.get_vision_encoder_info()
assert isinstance(encoder_info, PixtralHFEncoderInfo)
assert not isinstance(encoder_info, Mistral3HFEncoderInfo)
- assert encoder_info.get_image_size() == (
- processor.info.get_hf_config().vision_config.image_size
- )
+ vision_config = processor.info.get_hf_config().vision_config
+ assert isinstance(vision_config, PreTrainedConfig)
+ assert encoder_info.get_image_size() == vision_config.image_size
diff --git a/tests/models/multimodal/processing/test_mllama4.py b/tests/models/multimodal/processing/test_mllama4.py
index 6cf946eff279..362b327f53e7 100644
--- a/tests/models/multimodal/processing/test_mllama4.py
+++ b/tests/models/multimodal/processing/test_mllama4.py
@@ -3,6 +3,7 @@
"""Tests for mllama's multimodal preprocessing and profiling."""
import pytest
+import torch
from torch import prod
from transformers import Llama4Config
@@ -36,11 +37,13 @@ def test_profiling(model_id: str, max_model_len: int):
tokens_per_patch = ((image_size // patch_size) ** 2) // downsample_ratio
mm_data = mm_inputs["mm_kwargs"].get_data()
- chunks_per_image = prod(mm_data["patches_per_image"])
+ patches_per_image = mm_data["patches_per_image"]
+ aspect_ratios = mm_data["aspect_ratios"]
+ assert isinstance(patches_per_image, torch.Tensor)
+ assert isinstance(aspect_ratios, torch.Tensor)
+ chunks_per_image = prod(patches_per_image)
total_num_patches = chunks_per_image * tokens_per_patch
- num_tiles = (
- mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][1]
- ) # x-y separator tokens
+ num_tiles = aspect_ratios[0][0] * aspect_ratios[0][1] # x-y separator tokens
total_tokens = (
total_num_patches.item() + num_tiles.item() + 3
) # image start, image, image end
diff --git a/tests/models/multimodal/processing/test_molmo2.py b/tests/models/multimodal/processing/test_molmo2.py
index 77c6b8253d4f..1ecefb23a310 100644
--- a/tests/models/multimodal/processing/test_molmo2.py
+++ b/tests/models/multimodal/processing/test_molmo2.py
@@ -91,7 +91,9 @@ def test_dummy_video_num_frames_override_honors_min_of_two(
),
)
- video, _metadata = data["video"][0]
+ videos = data["video"]
+ assert isinstance(videos, list)
+ video, _metadata = videos[0]
assert video.shape[0] == expected_frames
diff --git a/tests/models/multimodal/processing/test_moondream3.py b/tests/models/multimodal/processing/test_moondream3.py
index a6284ae45fd1..f2c5d967f66d 100644
--- a/tests/models/multimodal/processing/test_moondream3.py
+++ b/tests/models/multimodal/processing/test_moondream3.py
@@ -98,6 +98,7 @@ def test_processor_pixel_values(
# Verify pixel_values shape
pixel_values = mm_data_result["pixel_values"]
+ assert isinstance(pixel_values, torch.Tensor)
assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W]
assert pixel_values.shape[2] == 3 # RGB channels
assert pixel_values.shape[3] == 378 # crop height
@@ -413,7 +414,9 @@ def test_processor_apply_accepts_non_pil_image_inputs(
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
mm_kwargs = processed_inputs["mm_kwargs"].get_data()
- assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
+ pixel_values = mm_kwargs["pixel_values"]
+ assert isinstance(pixel_values, torch.Tensor)
+ assert pixel_values.shape[2:] == (3, 378, 378)
class TestMoondream3TilingLogic:
diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py
index 294d53f23895..369f005dcc2d 100644
--- a/tests/models/multimodal/processing/test_moss_audio.py
+++ b/tests/models/multimodal/processing/test_moss_audio.py
@@ -34,6 +34,11 @@
from vllm.sequence import IntermediateTensors
+def _uninitialized_model() -> MossAudioModel:
+ # Each test sets only the fields needed by the method under test.
+ return object.__new__(MossAudioModel) # type: ignore[type-abstract]
+
+
class _Tokenizer:
def encode(self, text, **kwargs):
return [ord(char) for char in text]
@@ -294,7 +299,7 @@ def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
def test_moss_audio_supports_language_model_lora_only():
assert supports_lora(MossAudioModel)
- model = object.__new__(MossAudioModel)
+ model = _uninitialized_model()
assert isinstance(model, SupportsLoRA)
mapping = model.get_mm_mapping()
@@ -304,7 +309,7 @@ def test_moss_audio_supports_language_model_lora_only():
def test_moss_audio_error_paths():
- model = object.__new__(MossAudioModel)
+ model = _uninitialized_model()
with pytest.raises(ValueError, match="DeepStack audio token count mismatch"):
model._cache_deepstack_input_embeds(
inputs_embeds=torch.zeros(4, 8),
@@ -338,7 +343,7 @@ def test_moss_audio_validates_tp_config():
def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch():
- model = object.__new__(MossAudioModel)
+ model = _uninitialized_model()
with pytest.raises(ValueError, match="audio_data batch size"):
model._parse_and_validate_audio_input(
@@ -349,7 +354,7 @@ def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch():
@pytest.mark.parametrize("deepstack_scales", [(), (7, 11)])
def test_moss_audio_embed_multimodal_packs_by_audio(deepstack_scales):
- model = object.__new__(MossAudioModel)
+ model = _uninitialized_model()
model.audio_encoder = _FakeAudioEncoder(len(deepstack_scales))
model.audio_adapter = lambda hidden_states: hidden_states * 5
model.deepstack_audio_merger_list = [
@@ -395,7 +400,7 @@ class _FakeLanguageModel:
def embed_input_ids(self, input_ids):
return torch.zeros(input_ids.shape[0], 8)
- model = object.__new__(MossAudioModel)
+ model = _uninitialized_model()
model.language_model = _FakeLanguageModel()
model.deepstack_audio_merger_list = [object(), object()]
model.deepstack_input_embeds = None
@@ -442,7 +447,7 @@ def fake_lm(*args, _calls=calls, **kwargs):
return torch.ones(1, 1)
_patch_pp_group(monkeypatch, first=first)
- model = object.__new__(MossAudioModel)
+ model = _uninitialized_model()
torch.nn.Module.__init__(model)
model.language_model = fake_lm
cached = IntermediateTensors({"deepstack_input_embeds_0": torch.ones(3, 8)})
@@ -476,7 +481,7 @@ def fake_lm_non_first_rank(*args, **kwargs):
return torch.ones(1, 1)
_patch_pp_group(monkeypatch, first=False)
- model = object.__new__(MossAudioModel)
+ model = _uninitialized_model()
torch.nn.Module.__init__(model)
model.language_model = fake_lm_non_first_rank
model.deepstack_input_embeds = IntermediateTensors({})
@@ -509,7 +514,7 @@ def make_model(num_layers, deepstack_layers=None):
model.start_layer, model.end_layer = 0, num_layers
model.layers = torch.nn.ModuleList([AddOne() for _ in range(num_layers)])
model.norm = lambda hidden_states, residual: (hidden_states, residual)
- model._maybe_add_hidden_state = lambda aux, *args: aux
+ monkeypatch.setattr(model, "_maybe_add_hidden_state", lambda aux, *args: aux)
model.deepstack_inject_layer_indices = (
range(0) if deepstack_layers is None else deepstack_layers
)
diff --git a/tests/models/multimodal/processing/test_moss_transcribe_diarize.py b/tests/models/multimodal/processing/test_moss_transcribe_diarize.py
index 5a10e746d2a6..7f888b9446ac 100644
--- a/tests/models/multimodal/processing/test_moss_transcribe_diarize.py
+++ b/tests/models/multimodal/processing/test_moss_transcribe_diarize.py
@@ -19,9 +19,12 @@ def _question(request_prompt: str = "", hotwords: str | None = None) -> str:
hotwords=hotwords,
request_prompt=request_prompt,
)
- prompt = MossTranscribeDiarizeForConditionalGeneration.get_generation_prompt(
+ prompt_data = MossTranscribeDiarizeForConditionalGeneration.get_generation_prompt(
params
- )["prompt"]
+ )
+ assert isinstance(prompt_data, dict)
+ prompt = prompt_data.get("prompt")
+ assert isinstance(prompt, str)
return prompt.split("<|audio_end|>\n", 1)[1].split("<|im_end|>", 1)[0]
diff --git a/tests/models/multimodal/processing/test_nemotron_parse.py b/tests/models/multimodal/processing/test_nemotron_parse.py
index 019c06eb15a2..1acbe2eb84c0 100644
--- a/tests/models/multimodal/processing/test_nemotron_parse.py
+++ b/tests/models/multimodal/processing/test_nemotron_parse.py
@@ -37,12 +37,14 @@ def test_mm_device_do_normalize(width: int, height: int):
normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
+ assert isinstance(normalized_values, torch.Tensor)
ctx.model_config.multimodal_config.mm_device_do_normalize = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
+ assert isinstance(raw_values, torch.Tensor)
assert raw_values.dtype == torch.uint8
assert raw_values.shape == normalized_values.shape
@@ -71,7 +73,11 @@ def test_array_image_inputs(mm_device_do_normalize: bool, layout: str):
def pixel_values(img: object) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [img]})
- return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
+ values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()[
+ "pixel_values"
+ ]
+ assert isinstance(values, torch.Tensor)
+ return values
torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0)
@@ -88,5 +94,6 @@ def test_explicit_kwargs_without_device_normalize():
mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]})
values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
+ assert isinstance(values, torch.Tensor)
# The remote processor ignores these kwargs and still normalises on the host.
assert values.is_floating_point()
diff --git a/tests/models/multimodal/processing/test_nemotron_parse_35.py b/tests/models/multimodal/processing/test_nemotron_parse_35.py
index f8562c3e5220..11fbcc1ae45c 100644
--- a/tests/models/multimodal/processing/test_nemotron_parse_35.py
+++ b/tests/models/multimodal/processing/test_nemotron_parse_35.py
@@ -45,11 +45,15 @@ def test_causal_prefix_preserves_host_normalized_packed_images(
mm_items = processor.info.parse_mm_data({"image": [image]})
actual = processor([1], mm_items=mm_items)["mm_kwargs"].get_data()
- assert actual["pixel_values"].is_floating_point()
- assert actual["pixel_values"].shape[-1] == 3 * 16**2
+ pixel_values = actual["pixel_values"]
+ assert isinstance(pixel_values, torch.Tensor)
+ assert pixel_values.is_floating_point()
+ assert pixel_values.shape[-1] == 3 * 16**2
for name in ("pixel_values", "image_sizes", "content_sizes"):
+ value = actual[name]
+ assert isinstance(value, torch.Tensor)
torch.testing.assert_close(
- actual[name], expected[name].to(actual[name].dtype), rtol=0, atol=0
+ value, expected[name].to(value.dtype), rtol=0, atol=0
)
@@ -98,4 +102,4 @@ def test_causal_prefix_fields_config_handles_empty_cached_hf_inputs() -> None:
fields = processor._get_mm_fields_config(BatchFeature(), {})
- assert fields["pixel_values"].field.slices == []
+ assert getattr(fields["pixel_values"].field, "slices") == [] # noqa: B009
diff --git a/tests/models/multimodal/processing/test_nemotron_vl.py b/tests/models/multimodal/processing/test_nemotron_vl.py
index b1c1e11892c7..bc2e9782c01e 100644
--- a/tests/models/multimodal/processing/test_nemotron_vl.py
+++ b/tests/models/multimodal/processing/test_nemotron_vl.py
@@ -5,6 +5,7 @@
from collections.abc import Mapping
import pytest
+import torch
from PIL import Image
from transformers import PreTrainedConfig
@@ -77,7 +78,9 @@ def _run_check(
# Ensure we have the right number of placeholders per num_crops size
image_token_id = tokenizer.convert_tokens_to_ids("")
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
- pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
+ pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"]
+ assert isinstance(pixel_values, torch.Tensor)
+ pixel_shape = pixel_values.shape
print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape)
assert img_tok_count == 256 * total_expected_num_patches
assert pixel_shape[0] == total_expected_num_patches
diff --git a/tests/models/multimodal/processing/test_openvla.py b/tests/models/multimodal/processing/test_openvla.py
index 14eae4692a21..d88c716035bd 100644
--- a/tests/models/multimodal/processing/test_openvla.py
+++ b/tests/models/multimodal/processing/test_openvla.py
@@ -166,8 +166,10 @@ def test_openvla_image_processor_outputs_pixel_values() -> None:
output = processor(images=image)
- assert output["pixel_values"].shape == (1, 6, 224, 224)
- assert output["pixel_values"].dtype == torch.float32
+ pixel_values = output["pixel_values"]
+ assert isinstance(pixel_values, torch.Tensor)
+ assert pixel_values.shape == (1, 6, 224, 224)
+ assert pixel_values.dtype == torch.float32
def test_openvla_processing_info_token_counts() -> None:
@@ -198,6 +200,7 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
] == [(1, 1)]
assert content.full == [32000] * 256
+ assert callable(content.is_embed)
is_embed = content.is_embed(content.full)
assert is_embed.dtype == torch.bool
assert is_embed.tolist() == [True] * 256
diff --git a/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py b/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py
index fda9fa7aa8f4..e6c15dd6b383 100644
--- a/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py
+++ b/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py
@@ -48,7 +48,7 @@ def test_precomputed_vision_embeddings_preserve_items(model_version, modality):
torch.nn.Module.__init__(model)
model.visual = Mock(dtype=torch.float32, spatial_merge_size=2)
embeds = torch.arange(80, dtype=torch.float64).reshape(10, 8)
- outputs = model.embed_multimodal(
+ outputs = getattr(model, "embed_multimodal")( # noqa: B009
**{
f"{modality}_embeds": embeds,
f"{modality}_grid_thw": torch.tensor([[1, 2, 4], [2, 4, 4]]),
diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py
index 133880b08167..21f45d1f09dd 100644
--- a/tests/models/multimodal/processing/test_qwen2_vl.py
+++ b/tests/models/multimodal/processing/test_qwen2_vl.py
@@ -343,7 +343,9 @@ def test_processor_override(
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
- pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values"].shape
+ pixel_values = processed_inputs["mm_kwargs"].get_data()["pixel_values"]
+ assert isinstance(pixel_values, torch.Tensor)
+ pixel_shape = pixel_values.shape
assert img_tok_count == expected_toks_per_img * num_imgs
assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
@@ -371,6 +373,7 @@ def test_get_image_size_with_most_features(
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Qwen2VLProcessingInfo)
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
@@ -391,7 +394,9 @@ def test_get_image_size_with_most_features(
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
- grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist()
+ image_grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"]
+ assert isinstance(image_grid_thw, torch.Tensor)
+ grid_thw = image_grid_thw.tolist()
t, h, w = grid_thw[0]
tokens = (t * h * w) // (merge_size**2)
assert tokens < max_tokens
@@ -422,7 +427,9 @@ def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
for fps in (1.0, 30.0):
mm_data = _build_qwen2_5_vl_video_mm_data(num_frames=32, fps=fps)
processed = processor(prompt, mm_items=processor.info.parse_mm_data(mm_data))
- spg = processed["mm_kwargs"]["video"][0].get_data()["second_per_grid_ts"]
+ item = processed["mm_kwargs"]["video"][0]
+ assert item is not None
+ spg = item.get_data()["second_per_grid_ts"]
assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
@@ -458,6 +465,7 @@ def test_mm_device_do_normalize(
pixel_values_with_normalize = processed_inputs_with_normalize[
"mm_kwargs"
].get_data()["pixel_values"]
+ assert isinstance(pixel_values_with_normalize, torch.Tensor)
dtype = pixel_values_with_normalize.dtype
processed_inputs_without_normalize = processor(
@@ -468,6 +476,7 @@ def test_mm_device_do_normalize(
pixel_values_without_normalize = processed_inputs_without_normalize[
"mm_kwargs"
].get_data()["pixel_values"]
+ assert isinstance(pixel_values_without_normalize, torch.Tensor)
ctx.model_config.multimodal_config.mm_device_do_normalize = True
input_norm = build_mm_input_norm(ctx.model_config).to(device)
diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py
index 0c74a86ff000..f0c855efa984 100644
--- a/tests/models/multimodal/processing/test_qwen3_vl.py
+++ b/tests/models/multimodal/processing/test_qwen3_vl.py
@@ -87,6 +87,7 @@ def test_processor_partial_scoped_image_size(model_id: str) -> None:
limit_mm_per_prompt={"image": 1, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Qwen3VLProcessingInfo)
image = np.zeros((128, 128, 3), dtype=np.uint8)
min_pixels = 256 * 32 * 32
@@ -95,9 +96,11 @@ def test_processor_partial_scoped_image_size(model_id: str) -> None:
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [image]})
- return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
+ grid = processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"image_grid_thw"
]
+ assert isinstance(grid, torch.Tensor)
+ return grid
stock_grid = process({})
complete_grid = process({"images_kwargs": {"size": complete_size}})
@@ -127,6 +130,7 @@ def test_processor_partial_scoped_video_size(model_id: str) -> None:
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, Qwen3VLProcessingInfo)
min_pixels = 512 * 32 * 32
default_size = dict(processor.info.get_video_processor().size)
@@ -134,9 +138,11 @@ def test_processor_partial_scoped_video_size(model_id: str) -> None:
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
- return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
+ grid = processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
+ assert isinstance(grid, torch.Tensor)
+ return grid
stock_grid = process({})
complete_grid = process({"videos_kwargs": {"size": complete_size}})
@@ -302,6 +308,7 @@ def _probe_mm_token_budgets(
limit_mm_per_prompt={"image": 1, "video": 1},
)
info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
+ assert isinstance(info, Qwen3VLProcessingInfo)
video = info.get_max_video_tokens(
seq_len=500000, mm_counts={"video": 1, "image": 1}
)
@@ -381,7 +388,9 @@ def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
hf_processor_mm_kwargs=hf_kwargs,
)
- return single["mm_kwargs"].get_data()["video_grid_thw"][0]
+ grids = single["mm_kwargs"].get_data()["video_grid_thw"]
+ assert isinstance(grids, torch.Tensor)
+ return grids[0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
assert not torch.equal(expected_grids[0], expected_grids[1])
@@ -466,7 +475,9 @@ def test_processor_profiles_fewer_videos_than_configured_options(
profiled = processor.get_dummy_mm_inputs({"video": 1})
assert len(profiled["mm_placeholders"]["video"]) == 1
- assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
+ video_grid_thw = profiled["mm_kwargs"].get_data()["video_grid_thw"]
+ assert isinstance(video_grid_thw, torch.Tensor)
+ assert video_grid_thw.shape[0] == 1
def _build_video_embeds_mm_data(
@@ -584,7 +595,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
capped_dummy = capped.dummy_inputs.get_dummy_mm_data(
1024, {"video": 1}, MultiModalDummyOptions()
)
- capped_frames = capped_dummy["video"][0][0].shape[0]
+ capped_videos = capped_dummy["video"]
+ assert isinstance(capped_videos, list)
+ capped_frames = capped_videos[0][0].shape[0]
assert capped_frames == 16, (
f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}"
)
@@ -598,7 +611,9 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(
1024, {"video": 1}, MultiModalDummyOptions()
)
- uncapped_frames = uncapped_dummy["video"][0][0].shape[0]
+ uncapped_videos = uncapped_dummy["video"]
+ assert isinstance(uncapped_videos, list)
+ uncapped_frames = uncapped_videos[0][0].shape[0]
assert uncapped_frames == 2, (
f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}"
)
diff --git a/tests/models/multimodal/processing/test_smolvlm.py b/tests/models/multimodal/processing/test_smolvlm.py
index 02181d5d6fb6..f259eb7d4f02 100644
--- a/tests/models/multimodal/processing/test_smolvlm.py
+++ b/tests/models/multimodal/processing/test_smolvlm.py
@@ -5,6 +5,7 @@
import pytest
from transformers import SmolVLMConfig
+from vllm.model_executor.models.smolvlm import SmolVLMProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
@@ -40,6 +41,7 @@ def test_processor_override(
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, SmolVLMProcessingInfo)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py
index 73be3ab76c0b..573bdd4a8d8b 100644
--- a/tests/models/multimodal/processing/test_tensor_schema.py
+++ b/tests/models/multimodal/processing/test_tensor_schema.py
@@ -45,11 +45,10 @@
list[Image.Image] | list[np.ndarray] | list[tuple[np.ndarray, dict[str, Any]]]
)
AudioInput = list[tuple[np.ndarray, int]]
+ResizedData: TypeAlias = Image.Image | np.ndarray | list[Image.Image]
-def _resize_data(
- _data: Image.Image | np.ndarray, size_factor: float
-) -> Image.Image | np.ndarray:
+def _resize_data(_data: ResizedData, size_factor: float) -> ResizedData:
assert size_factor <= 1, "Size factor must be less than 1"
# Image input
if isinstance(_data, Image.Image):
@@ -75,7 +74,7 @@ def _resize_data(
def resize_mm_data(
data: ImageInput | VideoInput | AudioInput, size_factors: tuple[float, ...]
-) -> ImageInput | VideoInput | AudioInput:
+) -> list[ResizedData]:
size_factors = size_factors[: len(data)]
if is_list_of(data, (Image.Image, np.ndarray, list)):
return [_resize_data(d, s) for d, s in zip(data, size_factors)]
diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py
index 1c1204d8e5ce..b0dc9ae0cd57 100644
--- a/tests/models/multimodal/processing/test_transformers_audio.py
+++ b/tests/models/multimodal/processing/test_transformers_audio.py
@@ -4,6 +4,9 @@
import pytest
from vllm.config import ModelConfig
+from vllm.model_executor.models.transformers.multimodal import (
+ MultiModalProcessor as TransformersMultiModalProcessor,
+)
from vllm.multimodal import MULTIMODAL_REGISTRY
AUDIO_MODEL_SETTINGS = {
@@ -103,6 +106,7 @@ def test_audio_multimodal_processor(model_id):
audio_items = result.get("mm_kwargs", {}).get("audio", [])
assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}"
+ assert audio_items[0] is not None
item_keys = list(audio_items[0].keys())
has_features = "input_features" in item_keys or "input_values" in item_keys
assert has_features, (
@@ -142,6 +146,7 @@ def test_audio_fields_not_claimed_by_image():
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+ assert isinstance(mm_processor, TransformersMultiModalProcessor)
audio_keys = ["input_features", "input_features_mask"]
owned = mm_processor._partition_keys_by_modality(audio_keys, ["audio", "image"])
@@ -155,6 +160,7 @@ def test_unclaimed_fields_warn_rather_than_raise():
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+ assert isinstance(mm_processor, TransformersMultiModalProcessor)
owned = mm_processor._partition_keys_by_modality(
["input_features", "surprise_field"], ["audio", "image"]
diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py
index d45912a4b141..433a00942b80 100644
--- a/tests/models/multimodal/processing/test_transformers_image.py
+++ b/tests/models/multimodal/processing/test_transformers_image.py
@@ -3,6 +3,7 @@
from unittest.mock import patch
import pytest
+import torch
from vllm.assets.image import ImageAsset
from vllm.config import ModelConfig
@@ -113,8 +114,12 @@ def test_batch_padding_removed_from_image_items():
items = result["mm_kwargs"]["image"]
shapes = set()
for item in items:
- height, width = item["image_sizes"].data.flatten().tolist()
+ assert item is not None
+ image_sizes = item["image_sizes"].data
+ assert isinstance(image_sizes, torch.Tensor)
+ height, width = image_sizes.flatten().tolist()
pixel_values = item["pixel_values"].data
+ assert isinstance(pixel_values, torch.Tensor)
assert tuple(pixel_values.shape[-2:]) == (height, width)
shapes.add(tuple(pixel_values.shape))
@@ -219,6 +224,7 @@ def test_repeated_image_hits_the_processor_cache():
model_config = ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
)
+ assert model_config.multimodal_config is not None
model_config.multimodal_config.mm_processor_cache_gb = 4
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
cache = MultiModalProcessorOnlyCache(model_config)
@@ -303,5 +309,9 @@ def test_nested_image_fields_split_per_image():
items = result["mm_kwargs"]["image"]
assert len(items) == 2
for item in items:
+ assert item is not None
pixel_values = item["pixel_values"].data
- assert pixel_values.shape[1] == int(item["num_image_patches"].data)
+ num_image_patches = item["num_image_patches"].data
+ assert isinstance(pixel_values, torch.Tensor)
+ assert isinstance(num_image_patches, torch.Tensor)
+ assert pixel_values.shape[1] == int(num_image_patches)
diff --git a/tests/models/multimodal/processing/test_transformers_video.py b/tests/models/multimodal/processing/test_transformers_video.py
index 8d06843f06d3..00966495f6e0 100644
--- a/tests/models/multimodal/processing/test_transformers_video.py
+++ b/tests/models/multimodal/processing/test_transformers_video.py
@@ -2,9 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
+import torch
from transformers import Qwen3VLVideoProcessor
from vllm.config import ModelConfig
+from vllm.model_executor.models.transformers.multimodal import MultiModalProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
@@ -103,9 +105,14 @@ def test_video_multimodal_processor(model_id):
)
(item,) = result["mm_kwargs"]["video"]
- rows = item["pixel_values_videos"].data.shape[0]
+ assert item is not None
+ pixel_values_videos = item["pixel_values_videos"].data
+ assert isinstance(pixel_values_videos, torch.Tensor)
+ rows = pixel_values_videos.shape[0]
if "video_grid_thw" in item:
- assert rows == int(item["num_video_patches"].data)
+ num_video_patches = item["num_video_patches"].data
+ assert isinstance(num_video_patches, torch.Tensor)
+ assert rows == int(num_video_patches)
else:
assert rows == len(frames)
@@ -168,6 +175,8 @@ def test_video_fields_not_claimed_by_image():
(image_item,) = result["mm_kwargs"]["image"]
(video_item,) = result["mm_kwargs"]["video"]
+ assert image_item is not None
+ assert video_item is not None
assert {"pixel_values", "image_grid_thw"} <= set(image_item.keys())
assert {"pixel_values_videos", "video_grid_thw"} <= set(video_item.keys())
assert len(result["mm_placeholders"]["image"]) == 1
@@ -179,6 +188,7 @@ def test_repeated_video_hits_the_processor_cache():
model_config = ModelConfig(
model="Qwen/Qwen3-VL-2B-Instruct", model_impl="transformers"
)
+ assert model_config.multimodal_config is not None
model_config.multimodal_config.mm_processor_cache_gb = 4
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
cache = MultiModalProcessorOnlyCache(model_config)
@@ -222,7 +232,13 @@ def test_merged_token_fields_split_per_video():
)
items = result["mm_kwargs"]["video"]
- assert [len(item["video_compression_mask"].data) for item in items] == [128, 64]
+ assert all(item is not None for item in items)
+ assert [
+ len(item["video_compression_mask"].data) for item in items if item is not None
+ ] == [
+ 128,
+ 64,
+ ]
def test_video_sampled_by_processor():
@@ -262,6 +278,7 @@ def test_dummy_video_has_the_most_tokens(model_id):
ModelConfig(model=model_id, model_impl="transformers")
)
info = mm_processor.info
+ assert isinstance(info, MultiModalProcessingInfo)
seq_len = info.ctx.model_config.max_model_len
result = mm_processor.get_dummy_mm_inputs({"video": 1})
diff --git a/tests/models/multimodal/processing/test_unlimited_ocr.py b/tests/models/multimodal/processing/test_unlimited_ocr.py
index 87d2f2763119..4f5c08504725 100644
--- a/tests/models/multimodal/processing/test_unlimited_ocr.py
+++ b/tests/models/multimodal/processing/test_unlimited_ocr.py
@@ -6,6 +6,7 @@
from PIL import Image
from vllm.config import ModelConfig
+from vllm.model_executor.models.unlimited_ocr import UnlimitedOCRProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing.context import TimingContext
@@ -24,6 +25,7 @@ def test_profiling_tokens_cover_all_crop_grids():
model_config_kwargs={"max_model_len": 8192},
)
mm_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(mm_processor.info, UnlimitedOCRProcessingInfo)
hf_processor = mm_processor.info.get_hf_processor()
dummy_inputs = mm_processor.get_dummy_mm_inputs({"image": 1})
budget = dummy_inputs["mm_placeholders"]["image"][0].get_num_embeds()
@@ -48,6 +50,7 @@ def test_image_cache_identity_across_image_counts(use_uuid: bool, use_embeds: bo
)
processor = MULTIMODAL_REGISTRY.create_processor(config)
tokenizer = processor.info.get_tokenizer()
+ images: list[torch.Tensor] | list[Image.Image]
if use_embeds:
images = [torch.ones(8, config.get_hidden_size())] * 3
else:
diff --git a/tests/models/multimodal/processing/test_whisper.py b/tests/models/multimodal/processing/test_whisper.py
index 1627b48914c3..e7271fa92e5e 100644
--- a/tests/models/multimodal/processing/test_whisper.py
+++ b/tests/models/multimodal/processing/test_whisper.py
@@ -4,7 +4,9 @@
import numpy as np
import pytest
+import torch
+from vllm.model_executor.models.whisper import WhisperProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...utils import build_model_context
@@ -21,6 +23,7 @@ def test_audio_features_fit_encoder_window(
"""
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ assert isinstance(processor.info, WhisperProcessingInfo)
feature_extractor = processor.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
@@ -34,4 +37,5 @@ def test_audio_features_fit_encoder_window(
)
input_features = processed_inputs["mm_kwargs"].get_data()["input_features"]
+ assert isinstance(input_features, torch.Tensor)
assert input_features.shape[-1] == feature_extractor.nb_max_frames
diff --git a/tests/models/multimodal/test_mapping.py b/tests/models/multimodal/test_mapping.py
index a213091e3d09..ec92ff9eb854 100644
--- a/tests/models/multimodal/test_mapping.py
+++ b/tests/models/multimodal/test_mapping.py
@@ -342,7 +342,7 @@ def test_hf_model_weights_mapper(model_arch: str):
hf_dummy_model = create_dummy_model(model_id, model_arch)
hf_converted_weights = hf_dummy_model.named_parameters()
hf_converted_buffers = hf_dummy_model.named_buffers()
- mapper: WeightsMapper = model_cls.hf_to_vllm_mapper
+ mapper: WeightsMapper = getattr(model_cls, "hf_to_vllm_mapper") # noqa: B009
mapped_original_weights = mapper.apply(original_weights)
mapped_hf_converted_weights = mapper.apply(hf_converted_weights)
@@ -363,8 +363,7 @@ def test_hf_model_weights_mapper(model_arch: str):
config = hf_dummy_model.config
key = "tie_word_embeddings"
if getattr(config.get_text_config(), key, False) or getattr(config, key, False):
- mapped_tied_weights = mapper.apply((k, None) for k in tied)
- tied_weight_names = set(map(lambda x: x[0], mapped_tied_weights))
+ tied_weight_names = set(mapper.apply_list(tied))
ref_weight_names -= tied_weight_names
weights_missing = ref_weight_names - weight_names
diff --git a/tests/models/multimodal/test_mimo_v2_omni.py b/tests/models/multimodal/test_mimo_v2_omni.py
index deccace50d4e..9267217c278c 100644
--- a/tests/models/multimodal/test_mimo_v2_omni.py
+++ b/tests/models/multimodal/test_mimo_v2_omni.py
@@ -58,10 +58,9 @@ def test_window_attention_applies_sinks(vision_attn_env, num_kv_heads):
attn.sinks.data.normal_()
total = sum(SEQ_LENS)
- opts = dict(device="cuda", dtype=torch.bfloat16)
- q = torch.randn(total, NUM_HEADS, HEAD_DIM, **opts)
- k = torch.randn(total, num_kv_heads, HEAD_DIM, **opts)
- v = torch.randn(total, num_kv_heads, HEAD_DIM, **opts)
+ q = torch.randn(total, NUM_HEADS, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
+ k = torch.randn(total, num_kv_heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
+ v = torch.randn(total, num_kv_heads, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
cu_seqlens = torch.tensor(
[0, *torch.tensor(SEQ_LENS).cumsum(0).tolist()],
device="cuda",
diff --git a/tests/models/multimodal/test_nano_nemotron_vl.py b/tests/models/multimodal/test_nano_nemotron_vl.py
index fa7b1421defd..ee6f36893f9e 100644
--- a/tests/models/multimodal/test_nano_nemotron_vl.py
+++ b/tests/models/multimodal/test_nano_nemotron_vl.py
@@ -60,7 +60,7 @@ def test_nemotron_resized_normalization_matches_reference(
if not do_cpu_normalize:
assert output.dtype == torch.uint8
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
nn.Module.__init__(model)
model.input_norm = FusedMMInputNorm(
mean.flatten().tolist(), std.flatten().tolist(), 1.0 / 255.0
@@ -89,7 +89,7 @@ def test_nemotron_dynamic_device_normalization_matches_cpu_reference(
shape = (1, 7, 3 * 16 * 16)
pixels = torch.randint(0, 256, shape, dtype=torch.uint8)
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
nn.Module.__init__(model)
model.input_norm = FusedMMInputNorm(image_mean, image_std, 1.0 / 255.0).to(_DEVICE)
model.llm_dtype = torch.bfloat16
@@ -107,7 +107,7 @@ def test_nemotron_dynamic_device_normalization_matches_cpu_reference(
def test_nemotron_cpu_normalized_pixels_bypass_input_norm():
pixels = torch.randn(1, 3, 16, 16)
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
nn.Module.__init__(model)
model.input_norm = None
@@ -175,7 +175,7 @@ def test_nemotron_processor_defers_normalization_to_device(
# Normalizing on the device hands the encoder the same dtype as the CPU
# path. Values differ only where the uint8 resize rounds or clips the
# bicubic overshoot: a few elements by several levels, tiny on average.
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
nn.Module.__init__(model)
model.input_norm = FusedMMInputNorm(
config.norm_mean, config.norm_std, 1.0 / 255.0
@@ -202,9 +202,13 @@ def test_nemotron_processor_rejects_partial_normalization():
)
+def _uninitialized_model() -> NemotronH_Nano_VL_V2:
+ return object.__new__(NemotronH_Nano_VL_V2) # type: ignore[type-abstract]
+
+
@pytest.mark.parametrize("input_key", ["image_embeds", "video_embeds"])
def test_precomputed_multimodal_embeddings(input_key: str):
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
embeds = torch.randn(2, 4, 8)
outputs = model.embed_multimodal(**{input_key: embeds})
@@ -260,21 +264,8 @@ def load_weights(self, weights):
self.loaded_weights = list(weights)
-class _FakeTensor:
- """Sentinel stand-in for torch.Tensor in load_weights tests. Supports the
- .detach().clone() chain used by load_weights for buffered mm weights;
- both methods return self so identity (and the existing equality
- assertions) are preserved through cloning."""
-
- def detach(self):
- return self
-
- def clone(self):
- return self
-
-
def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
language_model = _LanguageModel()
object.__setattr__(model, "model_config", _ModelConfig())
object.__setattr__(model, "language_model", language_model)
@@ -283,15 +274,15 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
object.__setattr__(model, "vision_final_layernorm", _MissingMultiModalModule())
object.__setattr__(model, "sound_encoder", None)
- language_weight = object()
+ language_weight = torch.tensor(1.0)
model.load_weights(
[
("language_model.layers.0.weight", language_weight),
- ("mlp1.0.weight", object()),
- ("vision_model.radio_model.encoder.weight", object()),
- ("sound_encoder.encoder.weight", object()),
- ("vision_projector.vision_final_layernorm.weight", object()),
- ("vision_projector.vision_final_layernorm.bias", object()),
+ ("mlp1.0.weight", torch.empty(0)),
+ ("vision_model.radio_model.encoder.weight", torch.empty(0)),
+ ("sound_encoder.encoder.weight", torch.empty(0)),
+ ("vision_projector.vision_final_layernorm.weight", torch.empty(0)),
+ ("vision_projector.vision_final_layernorm.bias", torch.empty(0)),
]
)
@@ -299,7 +290,7 @@ def test_nano_nemotron_vl_skips_multimodal_weights_in_text_only_mode():
def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
language_model = _LanguageModel()
vision_model = _VisionModel()
object.__setattr__(model, "model_config", _ImageOnlyModelConfig())
@@ -309,8 +300,8 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
object.__setattr__(model, "vision_final_layernorm", None)
object.__setattr__(model, "sound_encoder", None)
- language_weight = object()
- vision_weight = _FakeTensor()
+ language_weight = torch.tensor(1.0)
+ vision_weight = torch.tensor([2.0, 3.0])
model.load_weights(
[
("language_model.layers.0.weight", language_weight),
@@ -319,13 +310,16 @@ def test_nano_nemotron_vl_loads_vision_weights_without_sound_encoder():
)
assert language_model.loaded_weights == [("layers.0.weight", language_weight)]
- assert vision_model.loaded_weights == [
- ("radio_model.encoder.weight", vision_weight)
- ]
+ assert len(vision_model.loaded_weights) == 1
+ name, loaded_weight = vision_model.loaded_weights[0]
+ assert name == "radio_model.encoder.weight"
+ assert isinstance(loaded_weight, torch.Tensor)
+ torch.testing.assert_close(loaded_weight, vision_weight)
+ assert loaded_weight is not vision_weight
def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights():
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
language_model = _LanguageModel()
vision_model = _VisionModel()
object.__setattr__(model, "model_config", _ImageOnlyModelConfig())
@@ -336,11 +330,11 @@ def test_nano_nemotron_vl_requires_sound_encoder_for_sound_weights():
object.__setattr__(model, "sound_encoder", None)
with pytest.raises(AssertionError):
- model.load_weights([("sound_encoder.encoder.weight", object())])
+ model.load_weights([("sound_encoder.encoder.weight", torch.empty(0))])
def _vision_norm_model(use_layernorm=True):
- model = object.__new__(NemotronH_Nano_VL_V2)
+ model = _uninitialized_model()
torch.nn.Module.__init__(model)
model.model_config = _ImageOnlyModelConfig()
model.language_model = _LanguageModel()
diff --git a/tests/models/multimodal/test_openpangu_vl.py b/tests/models/multimodal/test_openpangu_vl.py
index e8bab16d5d96..7267e893bcc4 100644
--- a/tests/models/multimodal/test_openpangu_vl.py
+++ b/tests/models/multimodal/test_openpangu_vl.py
@@ -22,14 +22,13 @@ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
class _OpenPanguVL(OpenPanguVLForConditionalGeneration):
+ language_model: _LanguageModel
+
def __init__(self, hidden_size: int) -> None:
nn.Module.__init__(self)
self.config = SimpleNamespace(image_token_id=41, video_token_id=42)
self.language_model = _LanguageModel(hidden_size)
- def get_language_model(self) -> _LanguageModel:
- return self.language_model
-
@pytest.mark.parametrize("device_type", ["cpu", "cuda"])
def test_get_input_embeddings_merges_image_and_video_embeddings(
@@ -40,7 +39,7 @@ def test_get_input_embeddings_merges_image_and_video_embeddings(
device = torch.device(device_type)
hidden_size = 4
- model = _OpenPanguVL(hidden_size).to(device)
+ model = _OpenPanguVL(hidden_size).to(device) # type: ignore[abstract]
input_ids = torch.tensor([7, 41, 41, 8, 42, 9], device=device)
multimodal_embeddings = (
torch.full((2, hidden_size), -1.0, device=device),
diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py
index 61dd74669a9d..d68cf06f7cdf 100755
--- a/tools/pre_commit/mypy.py
+++ b/tools/pre_commit/mypy.py
@@ -54,6 +54,7 @@
"tests/models/inkling",
"tests/models/kimi_k3",
"tests/models/language",
+ "tests/models/multimodal",
"tests/models/minimax_m3",
"tests/models/quantization",
"tests/models/qwen4_exp",
@@ -103,7 +104,6 @@
"tests/compile",
"tests/distributed",
"tests/models",
- "tests/models/multimodal",
"tests/v1",
"tests/v1/core",
"tests/v1/engine",
diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py
index 5a69787b73af..db3d86909a80 100644
--- a/vllm/config/multimodal.py
+++ b/vllm/config/multimodal.py
@@ -212,31 +212,31 @@ def _merge_mm_processor_kwargs(
class BaseDummyOptions:
"""Base options for generating dummy data during profiling."""
- count: int = Field(999, ge=0)
+ count: int = Field(default=999, ge=0)
@dataclass(config=ConfigDict(extra="forbid"))
class VideoDummyOptions(BaseDummyOptions):
"""Options for generating dummy video data during profiling."""
- num_frames: int | None = Field(None, gt=0)
- width: int | None = Field(None, gt=0)
- height: int | None = Field(None, gt=0)
+ num_frames: int | None = Field(default=None, gt=0)
+ width: int | None = Field(default=None, gt=0)
+ height: int | None = Field(default=None, gt=0)
@dataclass(config=ConfigDict(extra="forbid"))
class ImageDummyOptions(BaseDummyOptions):
"""Options for generating dummy image data during profiling."""
- width: int | None = Field(None, gt=0)
- height: int | None = Field(None, gt=0)
+ width: int | None = Field(default=None, gt=0)
+ height: int | None = Field(default=None, gt=0)
@dataclass(config=ConfigDict(extra="forbid"))
class AudioDummyOptions(BaseDummyOptions):
"""Options for generating dummy audio data during profiling."""
- length: int | None = Field(None, gt=0)
+ length: int | None = Field(default=None, gt=0)
@final
diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py
index 701c24a072d4..aabb85818fcd 100644
--- a/vllm/model_executor/models/interfaces.py
+++ b/vllm/model_executor/models/interfaces.py
@@ -736,7 +736,7 @@ class SupportsLoRA(Protocol):
packed_modules_mapping: dict[str, list[str]] = {}
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
lora_skip_prefixes: ClassVar[list[str]] = []
- lora_manager: "LoRAModelManager | None"
+ lora_manager: "LoRAModelManager | None" = None
# We can't use runtime_checkable with ClassVar for issubclass checks
diff --git a/vllm/model_executor/models/ming_v3_asr.py b/vllm/model_executor/models/ming_v3_asr.py
index 29e083407980..384e3c2bb06b 100644
--- a/vllm/model_executor/models/ming_v3_asr.py
+++ b/vllm/model_executor/models/ming_v3_asr.py
@@ -58,7 +58,7 @@
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
-from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
+from vllm.inputs import MultiModalDataDict, TokensPrompt
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
@@ -441,7 +441,7 @@ def get_speech_to_text_config(
)
@classmethod
- def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
+ def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> TokensPrompt:
if stt_params.task_type != "transcribe" or stt_params.to_language is not None:
raise ValueError("Ming V3 ASR does not support translation")
language = cls.validate_language(stt_params.language)