diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 7da3aa66c9c1..4d5d807692fd 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -264,8 +264,7 @@ def test_hf_token_cli_arg(cli_args, expected): }, ), ], -) -def test_media_io_kwargs_parser(arg, expected): +)def test_media_io_kwargs_parser(arg, expected): parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) if arg is None: args = parser.parse_args([]) @@ -275,6 +274,15 @@ def test_media_io_kwargs_parser(arg, expected): assert args.media_io_kwargs == expected +def test_max_video_size_mb_cli_arg(): + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + args = parser.parse_args(["--max-video-size-mb", "1024"]) + engine_args = EngineArgs.from_cli_args(args) + + assert args.max_video_size_mb == 1024 + assert engine_args.max_video_size_mb == 1024 + + @pytest.mark.parametrize( ("args", "expected"), [ diff --git a/tests/multimodal/media/test_connector.py b/tests/multimodal/media/test_connector.py index bee9d50ac1ce..217a792abcab 100644 --- a/tests/multimodal/media/test_connector.py +++ b/tests/multimodal/media/test_connector.py @@ -152,6 +152,20 @@ async def test_fetch_image_local_files(image_url: str): connector.fetch_image(f"file://{temp_dir}/../{os.path.basename(image_url)}") +def test_fetch_video_local_file_size_limit(): + with TemporaryDirectory() as temp_dir: + connector = MediaConnector( + allowed_local_media_path=temp_dir, + media_io_kwargs={"video": {"max_video_size_mb": 1}}, + ) + video_path = os.path.join(temp_dir, "oversized.mp4") + with open(video_path, "wb") as f: + f.write(b"0" * (2 * 1024 * 1024)) + + with pytest.raises(ValueError, match="exceeds the configured limit"): + connector.fetch_video(f"file://{video_path}") + + @pytest.mark.asyncio async def test_fetch_image_local_files_relative_allowed_path(tmp_path, monkeypatch): media_dir = tmp_path / "media" diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index 671abd7b0775..0c0a387dab75 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -216,6 +216,11 @@ def load_bytes( "video_backend should be consumed by VideoMediaIO, " "not passed to loader" ) + if "max_video_size_mb" in kwargs: + raise AssertionError( + "max_video_size_mb should be consumed by VideoMediaIO, " + "not passed to loader" + ) return FAKE_OUTPUT_1, {"received_kwargs": list(kwargs.keys())} with monkeypatch.context() as m: @@ -228,6 +233,7 @@ def load_bytes( imageio, num_frames=10, video_backend="test_reject_video_backend_kwarg", + max_video_size_mb=1, other_kwarg="should_pass_through", ) @@ -238,6 +244,16 @@ def load_bytes( assert "other_kwarg" in metadata["received_kwargs"] +def test_video_media_io_rejects_oversized_bytes(monkeypatch: pytest.MonkeyPatch): + with monkeypatch.context() as m: + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_reject_video_backend_kwarg") + + videoio = VideoMediaIO(ImageMediaIO(), num_frames=10, max_video_size_mb=1) + + with pytest.raises(ValueError, match="exceeds the configured limit"): + videoio.load_bytes(b"0" * (2 * 1024 * 1024)) + + def test_video_media_io_backend_env_var_fallback(monkeypatch: pytest.MonkeyPatch): """ Test that when video_backend kwarg is None or not provided, diff --git a/vllm/config/model.py b/vllm/config/model.py index 6e3fef0dcca7..3a343604104b 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -354,6 +354,7 @@ class ModelConfig: limit_mm_per_prompt: InitVar[dict[str, int | dict[str, int]] | None] = None enable_mm_embeds: InitVar[bool | None] = None media_io_kwargs: InitVar[dict[str, dict[str, Any]] | None] = None + max_video_size_mb: InitVar[int | None] = None mm_processor_kwargs: InitVar[dict[str, Any] | None] = None mm_processor_cache_gb: InitVar[float | None] = None mm_processor_cache_type: InitVar[MMCacheType | None] = None @@ -476,12 +477,12 @@ def _apply_dict_overrides( setattr(config, key, value) def __post_init__( - self, - # Multimodal config init vars + self, # Multimodal config init vars language_model_only: bool, limit_mm_per_prompt: dict[str, int | dict[str, int]] | None, enable_mm_embeds: bool | None, media_io_kwargs: dict[str, dict[str, Any]] | None, + max_video_size_mb: int | None, mm_processor_kwargs: dict[str, Any] | None, mm_processor_cache_gb: float | None, mm_processor_cache_type: MMCacheType | None, @@ -674,13 +675,20 @@ def __post_init__( self.original_max_model_len = self.max_model_len self.max_model_len = self.get_and_verify_max_len(self.max_model_len) - if self.is_encoder_decoder: mm_processor_cache_gb = 0 logger.info("Encoder-decoder model detected, disabling mm processor cache.") # Init multimodal config if needed if self._model_info.supports_multimodal: + if max_video_size_mb is not None and max_video_size_mb < 0: + raise ValueError("max_video_size_mb must be a non-negative integer") + if max_video_size_mb is not None: + media_io_kwargs = dict(media_io_kwargs or {}) + video_io_kwargs = dict(media_io_kwargs.get("video", {})) + video_io_kwargs["max_video_size_mb"] = max_video_size_mb + media_io_kwargs["video"] = video_io_kwargs + if ( mm_encoder_tp_mode == "data" and not self._model_info.supports_multimodal_encoder_tp_data @@ -2277,4 +2285,4 @@ def _get_and_verify_max_len( f"{msg} To allow overriding this maximum, set " f"the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1. {warning}" ) - return int(max_model_len) + return int(max_model_len) \ No newline at end of file diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 150d58cf1f74..07b01354e075 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -111,6 +111,9 @@ class MultiModalConfig: """Additional args passed to process media inputs, keyed by modalities. For example, to set num_frames for video, set `--media-io-kwargs '{"video": {"num_frames": 40} }'`""" + max_video_size_mb: int | None = None + """Maximum allowed video file size in MiB before decoding. + Set to 0 to disable the check.""" mm_processor_kwargs: dict[str, object] | None = None """Arguments to be forwarded to the model's processor for multi-modal data, e.g., image processor. Overrides for the multi-modal processor obtained diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index c7a9335bbeb0..980ac16cba1e 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -428,6 +428,7 @@ class EngineArgs: trust_remote_code: bool = ModelConfig.trust_remote_code allowed_local_media_path: str = ModelConfig.allowed_local_media_path allowed_media_domains: list[str] | None = ModelConfig.allowed_media_domains + max_video_size_mb: int | None = ModelConfig.max_video_size_mb download_dir: str | None = LoadConfig.download_dir safetensors_load_strategy: SafetensorsLoadStrategy | None = ( LoadConfig.safetensors_load_strategy @@ -1249,6 +1250,9 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: multimodal_group.add_argument( "--media-io-kwargs", **multimodal_kwargs["media_io_kwargs"] ) + multimodal_group.add_argument( + "--max-video-size-mb", **multimodal_kwargs["max_video_size_mb"] + ) multimodal_group.add_argument( "--mm-processor-kwargs", **multimodal_kwargs["mm_processor_kwargs"] ) @@ -1622,6 +1626,7 @@ def create_model_config(self) -> ModelConfig: trust_remote_code=self.trust_remote_code, allowed_local_media_path=self.allowed_local_media_path, allowed_media_domains=self.allowed_media_domains, + max_video_size_mb=self.max_video_size_mb, dtype=self.dtype, seed=self.seed, revision=self.revision, @@ -2713,4 +2718,4 @@ def _raise_unsupported_error(feature_name: str): f"{feature_name} is not supported. We recommend to " f"remove {feature_name} from your config." ) - raise NotImplementedError(msg) + raise NotImplementedError(msg) \ No newline at end of file diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py index 45ea4c2fdf45..92cda4f16def 100644 --- a/vllm/multimodal/media/video.py +++ b/vllm/multimodal/media/video.py @@ -50,6 +50,10 @@ def merge_kwargs( } merged = super().merge_kwargs(default_kwargs, runtime_kwargs) + if default_kwargs and "max_video_size_mb" in default_kwargs: + merged["max_video_size_mb"] = default_kwargs["max_video_size_mb"] + else: + merged.pop("max_video_size_mb", None) # fps and num_frames interact with each other, so if either is # overridden at request time, wipe the other from defaults to # avoid unintuitive cross-field interactions. @@ -84,10 +88,34 @@ def __init__( video_loader_backend = ( kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND ) + max_video_size_mb = kwargs.pop("max_video_size_mb", None) + if max_video_size_mb is not None and ( + not isinstance(max_video_size_mb, int) or max_video_size_mb < 0 + ): + raise ValueError("max_video_size_mb must be a non-negative integer") + self.max_video_size_mb = max_video_size_mb + self.max_video_size_bytes = ( + None + if max_video_size_mb in (None, 0) + else max_video_size_mb * 1024 * 1024 + ) self.kwargs = kwargs self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend) + def _validate_video_size(self, size_bytes: int, *, source: str) -> None: + if self.max_video_size_bytes is None or size_bytes <= self.max_video_size_bytes: + return + + size_mib = size_bytes / (1024 * 1024) + limit_mib = self.max_video_size_bytes / (1024 * 1024) + raise ValueError( + f"Refusing to load {source} because it is {size_mib:.2f} MiB, " + f"which exceeds the configured limit of {limit_mib:.2f} MiB. " + "Lower the input size or increase --max-video-size-mb." + ) + def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]: + self._validate_video_size(len(data), source="video payload") return self.video_loader.load_bytes( data, num_frames=self.num_frames, **self.kwargs ) @@ -96,6 +124,9 @@ def load_base64( self, media_type: str, data: str ) -> tuple[npt.NDArray, dict[str, Any]]: if media_type.lower() == "video/jpeg": + # Approximate the decoded payload size before processing frames. + raw_size = len(data) * 3 // 4 + self._validate_video_size(raw_size, source="video/jpeg base64 payload") load_frame = partial( self.image_io.load_base64, "image/jpeg", @@ -161,6 +192,10 @@ def load_base64( return self.load_bytes(pybase64.b64decode(data)) def load_file(self, filepath: Path) -> tuple[npt.NDArray, dict[str, Any]]: + self._validate_video_size( + filepath.stat().st_size, + source=f"local video file {filepath}", + ) with filepath.open("rb") as f: data = f.read() diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 874745e714f4..37a93864136e 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -923,21 +923,26 @@ def load_bytes( ) if backend == "opencv": - cap = cls.open_video_capture(data) - _check_frame_pixel_limit( - int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), - int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), - ) - source = cls._prepare_source(cls.get_video_metadata(cap)) - frame_idx = cls.compute_frames_index_to_sample( - source=source, target=target, **kwargs - ) - frames, valid = cls.read_frames( - cap, - frame_idx, - total_frames_num=source.total_frames_num, - frame_recovery=frame_recovery, - ) + try: + cap = cls.open_video_capture(data) + _check_frame_pixel_limit( + int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), + int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + source = cls._prepare_source(cls.get_video_metadata(cap)) + frame_idx = cls.compute_frames_index_to_sample( + source=source, target=target, **kwargs + ) + frames, valid = cls.read_frames( + cap, + frame_idx, + total_frames_num=source.total_frames_num, + frame_recovery=frame_recovery, + ) + except Exception as e: + raise ValueError( + f"Failed to load video with OpenCV backend: {e}" + ) from e elif backend == "pyav": assert not frame_recovery, ( "frame_recovery is only available for `opencv` backend"