diff --git a/tests/models/test_utils.py b/tests/models/test_utils.py index 8d47b4436575..295acf0dab18 100644 --- a/tests/models/test_utils.py +++ b/tests/models/test_utils.py @@ -7,12 +7,34 @@ from vllm.model_executor.models.utils import ( AutoWeightsLoader, _merge_multimodal_embeddings, + get_padded_num_video_frames, ) from vllm.platforms import current_platform DEVICE_TYPE = current_platform.device_type +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("num_frames", "temporal_patch_size", "expected"), + [ + (1, 2, 2), + (16, 4, 16), + (17, 4, 20), + (18, 4, 20), + (19, 4, 20), + (20, 4, 20), + (16.5, 4, 20), + ], +) +def test_get_padded_num_video_frames( + num_frames: int | float, temporal_patch_size: int, expected: int +): + padded_frames = get_padded_num_video_frames(num_frames, temporal_patch_size) + assert padded_frames == expected + assert isinstance(padded_frames, int) + + class ModuleWithBatchNorm(torch.nn.Module): def __init__(self): super().__init__() diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index d64a4ef51305..81d927c44770 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -111,6 +111,7 @@ from .utils import ( AutoWeightsLoader, WeightsMapper, + get_padded_num_video_frames, init_vllm_registered_model, maybe_prefix, ) @@ -1082,9 +1083,9 @@ def _get_vision_info( else: preprocessed_size = ImageSize(width=image_width, height=image_height) - # NOTE: Frames are padded to be divisible by `temporal_patch_size` - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294 - padded_num_frames = num_frames + num_frames % temporal_patch_size + # NOTE: Frames are padded to be divisible by `temporal_patch_size`. + # https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252 + padded_num_frames = get_padded_num_video_frames(num_frames, temporal_patch_size) grid_t = max(padded_num_frames // temporal_patch_size, 1) grid_h = preprocessed_size.height // patch_size diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index 125d7e71c7b5..dd3b7ad1770d 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -40,7 +40,12 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP from .qwen2_vl import Qwen2VisionTransformer -from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix +from .utils import ( + AutoWeightsLoader, + get_padded_num_video_frames, + init_vllm_registered_model, + maybe_prefix, +) logger = init_logger(__name__) @@ -408,9 +413,9 @@ def _get_vision_info( else: preprocessed_size = ImageSize(width=image_width, height=image_height) - # NOTE: Frames are padded to be divisible by `temporal_patch_size` - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294 - padded_num_frames = num_frames + num_frames % temporal_patch_size + # NOTE: Frames are padded to be divisible by `temporal_patch_size`. + # https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252 + padded_num_frames = get_padded_num_video_frames(num_frames, temporal_patch_size) grid_t = max(padded_num_frames // temporal_patch_size, 1) grid_h = preprocessed_size.height // patch_size diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index dd1fb892ad19..04352562b3b3 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -71,6 +71,7 @@ from .utils import ( AutoWeightsLoader, WeightsMapper, + get_padded_num_video_frames, init_vllm_registered_model, maybe_prefix, ) @@ -983,7 +984,7 @@ def _get_vision_info( else: preprocessed_size = ImageSize(width=image_width, height=image_height) - padded_num_frames = num_frames + num_frames % temporal_patch_size + padded_num_frames = get_padded_num_video_frames(num_frames, temporal_patch_size) grid_t = max(padded_num_frames // temporal_patch_size, 1) grid_h = preprocessed_size.height // patch_size diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py index 552e2a9e8f8d..4a467d1a65af 100644 --- a/vllm/model_executor/models/llava_onevision2.py +++ b/vllm/model_executor/models/llava_onevision2.py @@ -72,6 +72,7 @@ from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, + get_padded_num_video_frames, init_vllm_registered_model, maybe_prefix, ) @@ -1354,7 +1355,7 @@ def _get_vision_info( preprocessed = ImageSize(width=rw, height=rh) else: preprocessed = ImageSize(width=image_width, height=image_height) - padded_frames = num_frames + num_frames % temporal_patch_size + padded_frames = get_padded_num_video_frames(num_frames, temporal_patch_size) grid_t = max(padded_frames // temporal_patch_size, 1) grid_h = preprocessed.height // patch_size grid_w = preprocessed.width // patch_size diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index d0d9589ae1da..11be26edecbc 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -67,7 +67,13 @@ Qwen2_5_VLVideoPixelInputs, ) from .qwen2_vl import _create_qwen2vl_field_factory -from .utils import AutoWeightsLoader, IntermediateTensors, WeightsMapper, maybe_prefix +from .utils import ( + AutoWeightsLoader, + IntermediateTensors, + WeightsMapper, + get_padded_num_video_frames, + maybe_prefix, +) class MiMoVisionMLP(Qwen2_5_VisionMLP): @@ -715,7 +721,9 @@ def _get_vision_info( effective_frames = num_frames * tokens_per_second else: effective_frames = num_frames - padded_num_frames = effective_frames + effective_frames % temporal_patch_size + padded_num_frames = get_padded_num_video_frames( + effective_frames, temporal_patch_size + ) grid_t = max(padded_num_frames // temporal_patch_size, 1) grid_h = preprocessed_size.height // patch_size grid_w = preprocessed_size.width // patch_size diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 539f141cbaa7..e4963217d385 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -99,6 +99,7 @@ from .utils import ( AutoWeightsLoader, WeightsMapper, + get_padded_num_video_frames, init_vllm_registered_model, maybe_prefix, ) @@ -897,9 +898,9 @@ def _get_vision_info( else: preprocessed_size = ImageSize(width=image_width, height=image_height) - # NOTE: Frames are padded to be divisible by `temporal_patch_size` - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294 - padded_num_frames = num_frames + num_frames % temporal_patch_size + # NOTE: Frames are padded to be divisible by `temporal_patch_size`. + # https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252 + padded_num_frames = get_padded_num_video_frames(num_frames, temporal_patch_size) grid_t = max(padded_num_frames // temporal_patch_size, 1) grid_h = preprocessed_size.height // patch_size diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 6f4524400c05..8d35778a8c88 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -26,7 +26,7 @@ from vllm.model_executor.models.interfaces import supports_any_eagle from vllm.multimodal import NestedTensors from vllm.sequence import IntermediateTensors -from vllm.utils.math_utils import cdiv +from vllm.utils.math_utils import cdiv, round_up from vllm.utils.torch_utils import ( async_tensor_h2d, direct_register_custom_op, @@ -40,6 +40,13 @@ ShardId: TypeAlias = str | int | tuple[int, ...] +def get_padded_num_video_frames( + num_frames: int | float, temporal_patch_size: int +) -> int: + """Pad video frames to a multiple of the temporal patch size.""" + return int(round_up(num_frames, temporal_patch_size)) + + @dataclass class WeightsMapper: """Maps the name of each weight if they match the following patterns. diff --git a/vllm/utils/math_utils.py b/vllm/utils/math_utils.py index c681e708b5c1..7fe1533d4dc1 100644 --- a/vllm/utils/math_utils.py +++ b/vllm/utils/math_utils.py @@ -2,12 +2,22 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Math utility functions for vLLM.""" +from typing import overload + # Approximate value of 1/ln(2), used for log/exp base conversion # Best FP32 approximation: 1.4426950216 (hex 0x3FB8AA3B) RCP_LN2 = 1.4426950216 -def cdiv(a: int, b: int) -> int: +@overload +def cdiv(a: int, b: int) -> int: ... + + +@overload +def cdiv(a: float, b: int) -> float: ... + + +def cdiv(a: int | float, b: int) -> int | float: """Ceiling division.""" return -(a // -b) @@ -17,9 +27,17 @@ def next_power_of_2(n: int) -> int: return 1 if n < 1 else 1 << (n - 1).bit_length() -def round_up(x: int, y: int) -> int: +@overload +def round_up(x: int, y: int) -> int: ... + + +@overload +def round_up(x: float, y: int) -> float: ... + + +def round_up(x: int | float, y: int) -> int | float: """Round up x to the nearest multiple of y.""" - return ((x + y - 1) // y) * y + return cdiv(x, y) * y def round_down(x: int, y: int) -> int: