Validate processing kwargs with @strict from huggingface_hub

setup.py

@@ -114,7 +114,7 @@
     "GitPython<3.1.19",
     "hf-doc-builder>=0.3.0",
     "hf_xet",
-    "huggingface-hub==1.0.0.rc1",
+    "huggingface-hub==1.0.0.rc2",
     "importlib_metadata",
     "ipadic>=1.0.0,<2.0",
     "jinja2>=3.1.0",

src/transformers/dependency_versions_table.py

@@ -23,7 +23,7 @@
     "GitPython": "GitPython<3.1.19",
     "hf-doc-builder": "hf-doc-builder>=0.3.0",
     "hf_xet": "hf_xet",
-    "huggingface-hub": "huggingface-hub==1.0.0.rc1",
+    "huggingface-hub": "huggingface-hub==1.0.0.rc2",
     "importlib_metadata": "importlib_metadata",
     "ipadic": "ipadic>=1.0.0,<2.0",
     "jinja2": "jinja2>=3.1.0",

src/transformers/image_processing_utils_fast.py

@@ -15,7 +15,7 @@
 from collections.abc import Iterable
 from copy import deepcopy
 from functools import lru_cache, partial
-from typing import Any, Optional, TypedDict, Union
+from typing import Annotated, Any, Optional, TypedDict, Union
 
 import numpy as np
 
@@ -51,6 +51,7 @@
     logging,
 )
 from .utils.import_utils import is_rocm_platform
+from .utils.type_validators import TypedDictAdapter, device_validator, image_size_validator, tensor_type_validator
 
 
 if is_vision_available():
@@ -167,23 +168,23 @@ def divide_to_patches(
 
 class DefaultFastImageProcessorKwargs(TypedDict, total=False):
     do_resize: Optional[bool]
-    size: Optional[dict[str, int]]
+    size: Annotated[Optional[Union[int, list[int], tuple[int, ...], dict[str, int]]], image_size_validator()]
     default_to_square: Optional[bool]
-    resample: Optional[Union["PILImageResampling", "F.InterpolationMode"]]
+    resample: Optional[Union["PILImageResampling", "F.InterpolationMode", int]]
     do_center_crop: Optional[bool]
-    crop_size: Optional[dict[str, int]]
+    crop_size: Annotated[Optional[Union[int, list[int], tuple[int, ...], dict[str, int]]], image_size_validator()]
     do_rescale: Optional[bool]
     rescale_factor: Optional[Union[int, float]]
     do_normalize: Optional[bool]
-    image_mean: Optional[Union[float, list[float]]]
-    image_std: Optional[Union[float, list[float]]]
+    image_mean: Optional[Union[float, list[float], tuple[float, ...]]]
+    image_std: Optional[Union[float, list[float], tuple[float, ...]]]
     do_pad: Optional[bool]
-    pad_size: Optional[dict[str, int]]
+    pad_size: Annotated[Optional[Union[int, list[int], tuple[int, ...], dict[str, int]]], image_size_validator()]
     do_convert_rgb: Optional[bool]
-    return_tensors: Optional[Union[str, TensorType]]
-    data_format: Optional[ChannelDimension]
+    return_tensors: Annotated[Optional[Union[str, TensorType]], tensor_type_validator()]
+    data_format: Optional[Union[str, ChannelDimension]]
     input_data_format: Optional[Union[str, ChannelDimension]]
-    device: Optional["torch.device"]
+    device: Annotated[Optional[str], device_validator()]
     disable_grouping: Optional[bool]
 
 
@@ -737,6 +738,11 @@ def __call__(self, images: ImageInput, *args, **kwargs: Unpack[DefaultFastImageP
     def preprocess(self, images: ImageInput, *args, **kwargs: Unpack[DefaultFastImageProcessorKwargs]) -> BatchFeature:
         # args are not validated, but their order in the `preprocess` and `_preprocess` signatures must be the same
         validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self._valid_kwargs_names)
+
+        # Perform type validation on received kwargs
+        type_validator = TypedDictAdapter(self.valid_kwargs)
+        type_validator.validate_fields(**kwargs)
+
         # Set default kwargs from self. This ensures that if a kwarg is not provided
         # by the user, it gets its default value from the instance, or is set to None.
         for kwarg_name in self._valid_kwargs_names:

src/transformers/models/aria/modular_aria.py

@@ -38,7 +38,7 @@
 )
 from ...modeling_flash_attention_utils import FlashAttentionKwargs
 from ...modeling_utils import PreTrainedModel
-from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack
+from ...processing_utils import ImagesKwargs, MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack
 from ...tokenization_utils import PreTokenizedInput, TextInput
 from ...utils import TensorType, TransformersKwargs, auto_docstring, can_return_tuple, logging
 from ..auto import CONFIG_MAPPING, AutoConfig, AutoTokenizer
@@ -904,7 +904,15 @@ def get_number_of_image_patches(self, height: int, width: int, images_kwargs=Non
         return num_patches
 
 
+class AriaImagesKwargs(ImagesKwargs, total=False):
+    split_image: Optional[bool]
+    max_image_size: Optional[int]
+    min_image_size: Optional[int]
+
+
 class AriaProcessorKwargs(ProcessingKwargs, total=False):
+    images_kwargs: AriaImagesKwargs
+
     _defaults = {
         "text_kwargs": {
             "padding": False,

src/transformers/models/aria/processing_aria.py

@@ -24,13 +24,21 @@
 
 from ...image_processing_utils import BatchFeature
 from ...image_utils import ImageInput
-from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack
+from ...processing_utils import ImagesKwargs, MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack
 from ...tokenization_utils import PreTokenizedInput, TextInput
 from ...utils import TensorType
 from ..auto import AutoTokenizer
 
 
+class AriaImagesKwargs(ImagesKwargs, total=False):
+    split_image: Optional[bool]
+    max_image_size: Optional[int]
+    min_image_size: Optional[int]
+
+
 class AriaProcessorKwargs(ProcessingKwargs, total=False):
+    images_kwargs: AriaImagesKwargs
+
     _defaults = {
         "text_kwargs": {
             "padding": False,

src/transformers/models/csm/processing_csm.py

@@ -246,9 +246,7 @@ def __call__(
 
         text_kwargs = output_kwargs["text_kwargs"]
         audio_kwargs = output_kwargs["audio_kwargs"]
-        common_kwargs = output_kwargs["common_kwargs"]
-
-        return_tensors = common_kwargs.pop("return_tensors", None)
+        return_tensors = text_kwargs.get("return_tensors", None)
         if return_tensors != "pt":
             raise ValueError(f"{self.__class__.__name__} only supports `return_tensors='pt'`.")
 

src/transformers/models/deepseek_vl/image_processing_deepseek_vl_fast.py

@@ -42,7 +42,7 @@ class DeepseekVLFastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
         falls below this value after resizing.
     """
 
-    min_size: int
+    min_size: Optional[int]
 
 
 @auto_docstring

src/transformers/models/deepseek_vl_hybrid/image_processing_deepseek_vl_hybrid_fast.py

@@ -67,11 +67,11 @@ class DeepseekVLHybridFastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
         number of channels in the image. Can be overridden by the `high_res_image_std` parameter in the `preprocess` method.
     """
 
-    min_size: int
-    high_res_size: dict
-    high_res_resample: "PILImageResampling"
-    high_res_image_mean: list[float]
-    high_res_image_std: list[float]
+    min_size: Optional[int]
+    high_res_size: Optional[dict[str, int]]
+    high_res_resample: Optional[Union["PILImageResampling", "F.InterpolationMode", int]]
+    high_res_image_mean: Optional[Union[float, list[float], tuple[float, float, float]]]
+    high_res_image_std: Optional[Union[float, list[float], tuple[float, float, float]]]
 
 
 @auto_docstring

src/transformers/models/deepseek_vl_hybrid/modular_deepseek_vl_hybrid.py

@@ -752,11 +752,11 @@ class DeepseekVLHybridFastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
         number of channels in the image. Can be overridden by the `high_res_image_std` parameter in the `preprocess` method.
     """
 
-    min_size: int
-    high_res_size: dict
-    high_res_resample: "PILImageResampling"
-    high_res_image_mean: list[float]
-    high_res_image_std: list[float]
+    min_size: Optional[int]
+    high_res_size: Optional[dict[str, int]]
+    high_res_resample: Optional[Union["PILImageResampling", "F.InterpolationMode", int]]
+    high_res_image_mean: Optional[Union[float, list[float], tuple[float, float, float]]]
+    high_res_image_std: Optional[Union[float, list[float], tuple[float, float, float]]]
 
 
 class DeepseekVLHybridImageProcessorFast(DeepseekVLImageProcessorFast):

src/transformers/models/dia/processing_dia.py

@@ -55,7 +55,9 @@ class DiaProcessorKwargs(ProcessingKwargs, total=False):
             "generation": True,
             "sampling_rate": 44100,
         },
-        "common_kwargs": {"return_tensors": "pt"},
+        "common_kwargs": {
+            "return_tensors": "pt",
+        },
     }
 
 
@@ -111,9 +113,7 @@ def __call__(
 
         text_kwargs = output_kwargs["text_kwargs"]
         audio_kwargs = output_kwargs["audio_kwargs"]
-        common_kwargs = output_kwargs["common_kwargs"]
-
-        return_tensors = common_kwargs.pop("return_tensors", None)
+        return_tensors = output_kwargs["text_kwargs"]["return_tensors"]
         if return_tensors != "pt":
             raise ValueError(f"{self.__class__.__name__} only supports `return_tensors='pt'`.")
 

src/transformers/models/efficientnet/image_processing_efficientnet_fast.py

@@ -45,8 +45,8 @@ class EfficientNetFastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
             Normalize the image again with the standard deviation only for image classification if set to True.
     """
 
-    rescale_offset: bool
-    include_top: bool
+    rescale_offset: Optional[bool]
+    include_top: Optional[bool]
 
 
 @auto_docstring

src/transformers/models/eomt/image_processing_eomt_fast.py

@@ -59,19 +59,15 @@
 class EomtImageProcessorFastKwargs(DefaultFastImageProcessorKwargs):
     """
     do_split_image (`bool`, *optional*, defaults to `False`):
-            Whether to split the input images into overlapping patches for semantic segmentation. If set to `True`, the
-            input images will be split into patches of size `size["shortest_edge"]` with an overlap between patches.
-            Otherwise, the input images will be padded to the target size.
-    do_pad (`bool`, *optional*, defaults to `False`):
-            Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest
-            number of patches in the batch. Padding will be applied to the bottom and right with zeros.
+        Whether to split the input images into overlapping patches for semantic segmentation. If set to `True`, the
+        input images will be split into patches of size `size["shortest_edge"]` with an overlap between patches.
+        Otherwise, the input images will be padded to the target size.
     ignore_index (`int`, *optional*):
-            Label to be assigned to background pixels in segmentation maps. If provided, segmentation map pixels
-            denoted with 0 (background) will be replaced with `ignore_index`.
+        Label to be assigned to background pixels in segmentation maps. If provided, segmentation map pixels
+        denoted with 0 (background) will be replaced with `ignore_index`.
     """
 
-    do_split_image: bool
-    do_pad: bool
+    do_split_image: Optional[bool]
     ignore_index: Optional[int] = None
 
 

src/transformers/models/glm4v/processing_glm4v.py

@@ -34,7 +34,7 @@
 
 
 class Glm4vVideosProcessorKwargs(VideosKwargs, total=False):
-    fps: Union[list[float], float]
+    fps: Optional[Union[list[Union[float, int]], float, int]]
 
 
 class Glm4vImagesKwargs(ImagesKwargs):

src/transformers/models/glm4v/video_processing_glm4v.py

@@ -37,7 +37,7 @@
 
 
 class Glm4vVideoProcessorInitKwargs(VideosKwargs):
-    max_image_size: dict[str, int] = None
+    max_image_size: Optional[dict[str, int]] = None
     patch_size: Optional[int] = None
     temporal_patch_size: Optional[int] = None
     merge_size: Optional[int] = None

src/transformers/models/internvl/video_processing_internvl.py

@@ -33,7 +33,7 @@
 
 
 class InternVLVideoProcessorInitKwargs(VideosKwargs):
-    initial_shift: Union[bool, float, int]
+    initial_shift: Optional[Union[bool, float, int]]
 
 
 class InternVLVideoProcessor(BaseVideoProcessor):

src/transformers/models/janus/image_processing_janus_fast.py

@@ -53,7 +53,7 @@ class JanusFastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
         falls below this value after resizing.
     """
 
-    min_size: int
+    min_size: Optional[int]
 
 
 @auto_docstring

src/transformers/models/kosmos2/processing_kosmos2.py

@@ -33,9 +33,11 @@
     list[list[tuple[float, float, float]]],
 ]
 
+NestedList = list[Union[Optional[int], "NestedList"]]
+
 
 class Kosmos2ImagesKwargs(ImagesKwargs, total=False):
-    bboxes: Optional[list[float]]
+    bboxes: Optional[NestedList]
     num_image_tokens: Optional[int]
     first_image_token_id: Optional[int]
 

src/transformers/models/lfm2_vl/processing_lfm2_vl.py

@@ -21,6 +21,7 @@
     ImagesKwargs,
     ProcessingKwargs,
     ProcessorMixin,
+    TextKwargs,
     Unpack,
 )
 from ...tokenization_utils_base import BatchEncoding, TextInput
@@ -46,8 +47,13 @@ class Lfm2VlImagesKwargs(ImagesKwargs, total=False):
     return_row_col_info: Optional[bool]
 
 
+class Lfm2VlTextKwargs(TextKwargs, total=False):
+    use_image_special_tokens: Optional[bool]
+
+
 class Lfm2VlProcessorKwargs(ProcessingKwargs, total=False):
     images_kwargs: Lfm2VlImagesKwargs
+    text_kwargs: Lfm2VlTextKwargs
 
     _defaults = {
         "images_kwargs": {
@@ -75,8 +81,6 @@ class Lfm2VlProcessor(ProcessorMixin):
             An instance of [`PreTrainedTokenizerBase`]. This should correspond with the model's text model. The tokenizer is a required input.
         chat_template (`str`, *optional*):
             A Jinja template which will be used to convert lists of messages in a chat into a tokenizable string.
-        use_image_special_tokens (`bool`, *optional*, defaults to `True`):
-            Whether to use image special tokens or not when processing.
     """
 
     attributes = ["image_processor", "tokenizer"]
@@ -88,12 +92,10 @@ def __init__(
         image_processor,
         tokenizer,
         chat_template: Optional[str] = None,
-        use_image_special_tokens: Optional[bool] = True,
         **kwargs,
     ):
         self.image_token = tokenizer.image_token
         self.image_token_id = tokenizer.image_token_id
-        self.use_image_special_tokens = use_image_special_tokens
         self.image_start_token = tokenizer.image_start_token
         self.image_end_token = tokenizer.image_end_token
         self.image_thumbnail_token = tokenizer.image_thumbnail

src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py

@@ -92,8 +92,7 @@ def preprocess(self, images: ImageInput, **kwargs: Unpack[LlavaOnevisionFastImag
             batch_num_images = [1] * len(images)
         else:
             batch_num_images = [1]
-        kwargs["batch_num_images"] = batch_num_images
-        return super().preprocess(images, **kwargs)
+        return super().preprocess(images, batch_num_images, **kwargs)
 
     def _resize_for_patching(
         self,
@@ -218,6 +217,7 @@ def _pad_for_batching(
     def _preprocess(
         self,
         images: list["torch.Tensor"],
+        batch_num_images: list[int],
         do_resize: bool,
         size: SizeDict,
         image_grid_pinpoints: list[list[int]],
@@ -230,7 +230,6 @@ def _preprocess(
         image_mean: Optional[Union[float, list[float]]],
         image_std: Optional[Union[float, list[float]]],
         do_pad: bool,
-        batch_num_images: list[int],
         disable_grouping: Optional[bool],
         return_tensors: Optional[Union[str, TensorType]],
         **kwargs,