diff --git a/docker/transformers-pytorch-amd-gpu/Dockerfile b/docker/transformers-pytorch-amd-gpu/Dockerfile index 2c58491d686a..29f24e717de2 100644 --- a/docker/transformers-pytorch-amd-gpu/Dockerfile +++ b/docker/transformers-pytorch-amd-gpu/Dockerfile @@ -4,13 +4,25 @@ LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ - apt install -y --no-install-recommends git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-dev python3-pip python3-dev ffmpeg git-lfs && \ + apt install -y --no-install-recommends git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-dev python3-pip python3-dev ffmpeg git-lfs libjpeg-turbo8-dev libpng-dev zlib1g-dev && \ apt clean && \ rm -rf /var/lib/apt/lists/* RUN git lfs install RUN python3 -m pip install --no-cache-dir --upgrade pip numpy importlib-metadata setuptools wheel ninja pytesseract "itsdangerous<2.1.0" + +# Rebuild torchvision so decode_image has libjpeg and ROCm image ops stay on GPU. +RUN python3 -m pip install --no-cache-dir "setuptools<81" pybind11 +RUN TV_VERSION=$(python3 -c "import torchvision; print(torchvision.__version__.split('+')[0])") && \ + python3 -m pip uninstall -y torchvision && \ + git clone --depth 1 --branch "v${TV_VERSION}" https://github.com/pytorch/vision.git /tmp/vision && \ + cd /tmp/vision && \ + sed -i -E 's|list\(CSRS_DIR\.glob\("([^"]+\.cpp)"\)\)|[p for p in CSRS_DIR.glob("\1") if not p.name.endswith("_hip.cpp")]|g' setup.py && \ + FORCE_CUDA=1 TORCHVISION_USE_FFMPEG=0 TORCHVISION_USE_VIDEO_CODEC=0 \ + python3 -m pip install --no-cache-dir --no-build-isolation -v . && \ + cd / && rm -rf /tmp/vision + RUN python3 -m pip install --no-cache-dir --no-build-isolation git+https://github.com/facebookresearch/detectron2.git ARG REF=main diff --git a/examples/modular-transformers/configuration_duplicated_method.py b/examples/modular-transformers/configuration_duplicated_method.py index 9b4e8d029266..671fbd89b4a2 100644 --- a/examples/modular-transformers/configuration_duplicated_method.py +++ b/examples/modular-transformers/configuration_duplicated_method.py @@ -14,7 +14,7 @@ @auto_docstring(checkpoint="meta-duplicated_method/DuplicatedMethod-2-7b-hf") -@strict(accept_kwargs=True) +@strict class DuplicatedMethodConfig(PreTrainedConfig): r""" ```python diff --git a/examples/modular-transformers/configuration_my_new_model.py b/examples/modular-transformers/configuration_my_new_model.py index a4c369fca146..a5e533c8e784 100644 --- a/examples/modular-transformers/configuration_my_new_model.py +++ b/examples/modular-transformers/configuration_my_new_model.py @@ -14,7 +14,7 @@ @auto_docstring(checkpoint="meta-my_new_model/MyNewModel-2-7b-hf") -@strict(accept_kwargs=True) +@strict class MyNewModelConfig(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`MyNewModelModel`]. It is used to instantiate an MyNewModel diff --git a/examples/modular-transformers/configuration_my_new_model2.py b/examples/modular-transformers/configuration_my_new_model2.py index eda7a1c1b7c2..635beadc20ee 100644 --- a/examples/modular-transformers/configuration_my_new_model2.py +++ b/examples/modular-transformers/configuration_my_new_model2.py @@ -13,7 +13,7 @@ @auto_docstring(checkpoint="meta-my_new_model2/MyNewModel2-2-7b-hf") -@strict(accept_kwargs=True) +@strict class MyNewModel2Config(PreTrainedConfig): r""" This is the configuration class to store the configuration of a [`GemmaModel`]. It is used to instantiate an Gemma diff --git a/examples/modular-transformers/configuration_new_model.py b/examples/modular-transformers/configuration_new_model.py index f27ab4917ee4..bbdc8a6bdd8c 100644 --- a/examples/modular-transformers/configuration_new_model.py +++ b/examples/modular-transformers/configuration_new_model.py @@ -13,7 +13,7 @@ @auto_docstring(checkpoint="google/new_model-7b") -@strict(accept_kwargs=True) +@strict class NewModelConfig(PreTrainedConfig): r""" use_bidirectional_attention (`bool`, *optional*): diff --git a/examples/modular-transformers/image_processing_new_imgproc_model.py b/examples/modular-transformers/image_processing_new_imgproc_model.py index 5df950414eb4..91efd59673b2 100644 --- a/examples/modular-transformers/image_processing_new_imgproc_model.py +++ b/examples/modular-transformers/image_processing_new_imgproc_model.py @@ -4,277 +4,24 @@ # the file from the modular. If any change should be done, please apply the change to the # modular_new_imgproc_model.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 -import numpy as np import torch -from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict -from ...image_transforms import convert_to_rgb, resize, to_channel_dimension_format -from ...image_utils import ( - OPENAI_CLIP_MEAN, - OPENAI_CLIP_STD, - ChannelDimension, - ImageInput, - PILImageResampling, - infer_channel_dimension_format, - is_scaled_image, - make_flat_list_of_images, - to_numpy_array, - valid_images, - validate_preprocess_arguments, -) -from ...utils import TensorType, filter_out_non_signature_kwargs, is_vision_available, logging - - -if is_vision_available(): - import PIL - - -logger = logging.get_logger(__name__) - - -class ImgprocModelImageProcessor(BaseImageProcessor): - r""" - Constructs a IMGPROC_MODEL image processor. - - Args: - do_resize (`bool`, *optional*, defaults to `True`): - Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the - `do_resize` parameter in the `preprocess` method. - size (`dict`, *optional*, defaults to `{"height": 384, "width": 384}`): - Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess` - method. - resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`): - Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be - overridden by the `resample` parameter in the `preprocess` method. - do_rescale (`bool`, *optional*, defaults to `True`): - Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the - `do_rescale` parameter in the `preprocess` method. - rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): - Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be - overridden by the `rescale_factor` parameter in the `preprocess` method. - do_normalize (`bool`, *optional*, defaults to `True`): - Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` - method. Can be overridden by the `do_normalize` parameter in the `preprocess` method. - image_mean (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): - Mean to use if normalizing the image. This is a float or list of floats the length of the number of - channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be - overridden by the `image_mean` parameter in the `preprocess` method. - image_std (`float` or `list[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): - Standard deviation to use if normalizing the image. This is a float or list of floats the length of the - number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. - Can be overridden by the `image_std` parameter in the `preprocess` method. - do_convert_rgb (`bool`, *optional*, defaults to `True`): - Whether to convert the image to RGB. - """ - - model_input_names = ["pixel_values"] - - def __init__( - self, - do_resize: bool = True, - size: dict[str, int] | None = None, - resample: PILImageResampling = PILImageResampling.BICUBIC, - do_rescale: bool = True, - rescale_factor: int | float = 1 / 255, - do_normalize: bool = True, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, - do_convert_rgb: bool = True, - **kwargs, - ) -> None: - super().__init__(**kwargs) - size = size if size is not None else {"height": 384, "width": 384} - size = get_size_dict(size, default_to_square=True) - - self.do_resize = do_resize - self.size = size - self.resample = resample - self.do_rescale = do_rescale - self.rescale_factor = rescale_factor - self.do_normalize = do_normalize - self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN - self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD - self.do_convert_rgb = do_convert_rgb - - def resize( - self, - image: np.ndarray, - size: dict[str, int], - resample: PILImageResampling = PILImageResampling.BICUBIC, - data_format: str | ChannelDimension | None = None, - input_data_format: str | ChannelDimension | None = None, - **kwargs, - ) -> np.ndarray: - """ - Resize an image to `(size["height"], size["width"])`. - - Args: - image (`np.ndarray`): - Image to resize. - size (`dict[str, int]`): - Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. - resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): - `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`. - data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the output image. If unset, the channel dimension format of the input - image is used. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - - Returns: - `np.ndarray`: The resized image. - """ - size = get_size_dict(size) - if "height" not in size or "width" not in size: - raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") - output_size = (size["height"], size["width"]) - return resize( - image, - size=output_size, - resample=resample, - data_format=data_format, - input_data_format=input_data_format, - **kwargs, - ) - - @filter_out_non_signature_kwargs() - def preprocess( - self, - images: ImageInput, - do_resize: bool | None = None, - size: dict[str, int] | None = None, - resample: PILImageResampling | None = None, - do_rescale: bool | None = None, - rescale_factor: float | None = None, - do_normalize: bool | None = None, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, - return_tensors: str | TensorType | None = None, - do_convert_rgb: bool | None = None, - data_format: ChannelDimension = ChannelDimension.FIRST, - input_data_format: str | ChannelDimension | None = None, - ) -> PIL.Image.Image: - """ - Preprocess an image or batch of images. - - Args: - images (`ImageInput`): - Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If - passing in images with pixel values between 0 and 1, set `do_rescale=False`. - do_resize (`bool`, *optional*, defaults to `self.do_resize`): - Whether to resize the image. - size (`dict[str, int]`, *optional*, defaults to `self.size`): - Controls the size of the image after `resize`. The shortest edge of the image is resized to - `size["shortest_edge"]` whilst preserving the aspect ratio. If the longest edge of this resized image - is > `int(size["shortest_edge"] * (1333 / 800))`, then the image is resized again to make the longest - edge equal to `int(size["shortest_edge"] * (1333 / 800))`. - resample (`PILImageResampling`, *optional*, defaults to `self.resample`): - Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. - do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): - Whether to rescale the image values between [0 - 1]. - rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): - Rescale factor to rescale the image by if `do_rescale` is set to `True`. - do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): - Whether to normalize the image. - image_mean (`float` or `list[float]`, *optional*, defaults to `self.image_mean`): - Image mean to normalize the image by if `do_normalize` is set to `True`. - image_std (`float` or `list[float]`, *optional*, defaults to `self.image_std`): - Image standard deviation to normalize the image by if `do_normalize` is set to `True`. - do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): - Whether to convert the image to RGB. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): - The channel dimension format for the output image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - Unset: Use the channel dimension format of the input image. - input_data_format (`ChannelDimension` or `str`, *optional*): - The channel dimension format for the input image. If unset, the channel dimension format is inferred - from the input image. Can be one of: - - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. - - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. - - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. - """ - do_resize = do_resize if do_resize is not None else self.do_resize - resample = resample if resample is not None else self.resample - do_rescale = do_rescale if do_rescale is not None else self.do_rescale - rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor - do_normalize = do_normalize if do_normalize is not None else self.do_normalize - image_mean = image_mean if image_mean is not None else self.image_mean - image_std = image_std if image_std is not None else self.image_std - do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb - - size = size if size is not None else self.size - size = get_size_dict(size, default_to_square=False) - images = self.fetch_images(images) - images = make_flat_list_of_images(images) - - if not valid_images(images): - raise ValueError("Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, or torch.Tensor") - - validate_preprocess_arguments( - do_rescale=do_rescale, - rescale_factor=rescale_factor, - do_normalize=do_normalize, - image_mean=image_mean, - image_std=image_std, - do_resize=do_resize, - size=size, - resample=resample, - ) - # PIL RGBA images are converted to RGB - if do_convert_rgb: - images = [convert_to_rgb(image) for image in images] - - # All transformations expect numpy arrays. - images = [to_numpy_array(image) for image in images] - - if do_rescale and is_scaled_image(images[0]): - logger.warning_once( - "It looks like you are trying to rescale already rescaled images. If the input" - " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." - ) - - if input_data_format is None: - # We assume that all images have the same channel dimension format. - input_data_format = infer_channel_dimension_format(images[0]) - - if do_resize: - images = [ - self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) - for image in images - ] - - if do_rescale: - images = [ - self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) - for image in images - ] - - if do_normalize: - images = [ - self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) - for image in images - ] - - images = [ - to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images - ] - - encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) - - return encoded_outputs +from ...image_processing_backends import TorchvisionBackend +from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, PILImageResampling +from ...utils import auto_docstring + + +@auto_docstring +class ImgprocModelImageProcessor(TorchvisionBackend): + resample = PILImageResampling.BICUBIC + image_mean = OPENAI_CLIP_MEAN + image_std = OPENAI_CLIP_STD + size = {"height": 384, "width": 384} + default_to_square = True + do_resize = True + do_rescale = True + do_normalize = True + do_convert_rgb = True def new_image_processing_method(self, pixel_values: torch.FloatTensor): return pixel_values / 2 diff --git a/examples/modular-transformers/modeling_dummy_bert.py b/examples/modular-transformers/modeling_dummy_bert.py index e0c4a03556f6..c18b9eef839f 100644 --- a/examples/modular-transformers/modeling_dummy_bert.py +++ b/examples/modular-transformers/modeling_dummy_bert.py @@ -214,14 +214,12 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor]: # determine input shapes - bsz, tgt_len = hidden_states.shape[:-1] - src_len = encoder_hidden_states.shape[1] + input_shape = hidden_states.shape[:-1] - q_input_shape = (bsz, tgt_len, -1, self.attention_head_size) - kv_input_shape = (bsz, src_len, -1, self.attention_head_size) + hidden_shape = (*input_shape, -1, self.attention_head_size) # get query proj - query_layer = self.query(hidden_states).view(*q_input_shape).transpose(1, 2) + query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2) is_updated = past_key_values.is_updated.get(self.layer_idx) if past_key_values is not None else False if past_key_values is not None and is_updated: @@ -229,8 +227,9 @@ def forward( key_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].keys value_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].values else: - key_layer = self.key(encoder_hidden_states).view(*kv_input_shape).transpose(1, 2) - value_layer = self.value(encoder_hidden_states).view(*kv_input_shape).transpose(1, 2) + kv_shape = (*encoder_hidden_states.shape[:-1], -1, self.attention_head_size) + key_layer = self.key(encoder_hidden_states).view(kv_shape).transpose(1, 2) + value_layer = self.value(encoder_hidden_states).view(kv_shape).transpose(1, 2) if past_key_values is not None: # save all states to the cache @@ -254,7 +253,7 @@ def forward( scaling=self.scaling, **kwargs, ) - attn_output = attn_output.reshape(bsz, tgt_len, -1).contiguous() + attn_output = attn_output.reshape(*input_shape, -1).contiguous() return attn_output, attn_weights diff --git a/examples/modular-transformers/modeling_from_uppercase_model.py b/examples/modular-transformers/modeling_from_uppercase_model.py index 31b818cf3e80..5cb9b8db4cab 100644 --- a/examples/modular-transformers/modeling_from_uppercase_model.py +++ b/examples/modular-transformers/modeling_from_uppercase_model.py @@ -65,15 +65,16 @@ def forward( ) -> tuple[torch.Tensor, torch.Tensor | None]: """Input shape: Batch x Time x Channel""" - batch_size, seq_length, embed_dim = hidden_states.shape + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) queries = self.q_proj(hidden_states) keys = self.k_proj(hidden_states) values = self.v_proj(hidden_states) - queries = queries.view(batch_size, seq_length, -1, self.head_dim).transpose(1, 2) - keys = keys.view(batch_size, seq_length, -1, self.head_dim).transpose(1, 2) - values = values.view(batch_size, seq_length, -1, self.head_dim).transpose(1, 2) + queries = queries.view(hidden_shape).transpose(1, 2) + keys = keys.view(hidden_shape).transpose(1, 2) + values = values.view(hidden_shape).transpose(1, 2) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -90,7 +91,7 @@ def forward( **kwargs, ) - attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous() + attn_output = attn_output.reshape(*input_shape, -1).contiguous() attn_output = self.out_proj(attn_output) return attn_output, attn_weights diff --git a/examples/modular-transformers/modeling_multimodal2.py b/examples/modular-transformers/modeling_multimodal2.py index f15a440b7f03..f44e74a18d8e 100644 --- a/examples/modular-transformers/modeling_multimodal2.py +++ b/examples/modular-transformers/modeling_multimodal2.py @@ -68,15 +68,16 @@ def forward( ) -> tuple[torch.Tensor, torch.Tensor | None]: """Input shape: Batch x Time x Channel""" - batch_size, seq_length, embed_dim = hidden_states.shape + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, self.head_dim) queries = self.q_proj(hidden_states) keys = self.k_proj(hidden_states) values = self.v_proj(hidden_states) - queries = queries.view(batch_size, seq_length, -1, self.head_dim).transpose(1, 2) - keys = keys.view(batch_size, seq_length, -1, self.head_dim).transpose(1, 2) - values = values.view(batch_size, seq_length, -1, self.head_dim).transpose(1, 2) + queries = queries.view(hidden_shape).transpose(1, 2) + keys = keys.view(hidden_shape).transpose(1, 2) + values = values.view(hidden_shape).transpose(1, 2) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -93,7 +94,7 @@ def forward( **kwargs, ) - attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous() + attn_output = attn_output.reshape(*input_shape, -1).contiguous() attn_output = self.out_proj(attn_output) return attn_output, attn_weights @@ -168,20 +169,6 @@ def forward( attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutput: - r""" - Args: - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - """ hidden_states = inputs_embeds for encoder_layer in self.layers: hidden_states = encoder_layer( @@ -200,6 +187,12 @@ class Multimodal2VisionPreTrainedModel(PreTrainedModel): config: Multimodal2Config base_model_prefix = "multimodal2_vision" input_modalities = ("image", "text") + _no_split_modules = [ + "Multimodal2VisionTextEmbeddings", + "Multimodal2VisionEncoderLayer", + "Multimodal2VisionVisionEmbeddings", + ] + supports_gradient_checkpointing = True _supports_sdpa = True _supports_flash_attn = True @@ -300,15 +293,20 @@ def forward(self, pixel_values: torch.FloatTensor, interpolate_pos_encoding=Fals return embeddings -class Multimodal2VisionTransformer(Multimodal2VisionPreTrainedModel): +@auto_docstring( + custom_intro=""" + The vision model from MULTIMODAL2 without any head or projection on top. + """ +) +class Multimodal2VisionModel(Multimodal2VisionPreTrainedModel): config: Multimodal2VisionConfig main_input_name = "pixel_values" input_modalities = ("image",) - _no_split_modules = ["CLIPEncoderLayer"] + _input_embed_layer = "patch_embedding" + _no_split_modules = ["Multimodal2VisionEncoderLayer"] def __init__(self, config): super().__init__(config) - self.config = config embed_dim = config.hidden_size self.embeddings = Multimodal2VisionEmbeddings(config) @@ -325,54 +323,6 @@ def forward( pixel_values: torch.FloatTensor | None = None, interpolate_pos_encoding: bool | None = False, **kwargs: Unpack[TransformersKwargs], - ) -> BaseModelOutputWithPooling: - if pixel_values is None: - raise ValueError("You have to specify pixel_values") - - hidden_states = self.embeddings(pixel_values, interpolate_pos_encoding=interpolate_pos_encoding) - hidden_states = self.pre_layrnorm(hidden_states) - - encoder_outputs: BaseModelOutput = self.encoder( - inputs_embeds=hidden_states, - **kwargs, - ) - - last_hidden_state = encoder_outputs.last_hidden_state - pooled_output = last_hidden_state[:, 0, :] - pooled_output = self.post_layernorm(pooled_output) - - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=pooled_output, - ) - - -@auto_docstring( - custom_intro=""" - The vision model from MULTIMODAL2 without any head or projection on top. - """ -) -class Multimodal2VisionModel(Multimodal2VisionPreTrainedModel): - config: Multimodal2VisionConfig - main_input_name = "pixel_values" - input_modalities = ("image",) - _no_split_modules = ["Multimodal2VisionEncoderLayer"] - - def __init__(self, config: Multimodal2VisionConfig): - super().__init__(config) - self.vision_model = Multimodal2VisionTransformer(config) - # Initialize weights and apply final processing - self.post_init() - - def get_input_embeddings(self) -> nn.Module: - return self.vision_model.embeddings.patch_embedding - - @auto_docstring - def forward( - self, - pixel_values: torch.FloatTensor | None = None, - interpolate_pos_encoding: bool = False, - **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPooling: r""" Example: @@ -396,9 +346,19 @@ def forward( >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled CLS states ```""" + hidden_states = self.embeddings(pixel_values, interpolate_pos_encoding=interpolate_pos_encoding) + hidden_states = self.pre_layrnorm(hidden_states) - return self.vision_model( - pixel_values=pixel_values, - interpolate_pos_encoding=interpolate_pos_encoding, + encoder_outputs: BaseModelOutput = self.encoder( + inputs_embeds=hidden_states, **kwargs, ) + + last_hidden_state = encoder_outputs.last_hidden_state + pooled_output = last_hidden_state[:, 0, :] + pooled_output = self.post_layernorm(pooled_output) + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + ) diff --git a/examples/modular-transformers/modeling_new_task_model.py b/examples/modular-transformers/modeling_new_task_model.py index 6e739fa0dbf4..ff6e666a804e 100644 --- a/examples/modular-transformers/modeling_new_task_model.py +++ b/examples/modular-transformers/modeling_new_task_model.py @@ -100,42 +100,29 @@ class NewTaskModelPreTrainedModel(PreTrainedModel): _supports_attention_backend = True -def token_type_ids_mask_function( - token_type_ids: torch.Tensor | None, - image_group_ids: torch.Tensor | None, -) -> Callable | None: +def token_type_ids_mask_function(group_ids: torch.Tensor) -> Callable: """ This function adds the correct offsets to the `q_idx` and `kv_idx` as the torch API can only accept lengths, not start and end indices. + Args: + group_ids (`torch.Tensor`): + A tensor of shape `(bs, len)` assigning each token to a vision group. Tokens with the same group + come from the same input image. Text is denoted by `-1`. """ - # Do not return an additional mask in this case - if token_type_ids is None: - return None def inner_mask(batch_idx: int, head_idx: int, q_idx: int, kv_idx: int) -> bool: - # If it's 1 for both query and key/value, we are in an image block - # NOTE: static cache shape goes beyond input seq length, while token_type_ids.shape[1] == input seq length - # Since vmap doesn't support `if statement` we workaround it with `torch.where` - safe_q_idx = torch.where(q_idx < token_type_ids.shape[1], q_idx, 0) - safe_kv_idx = torch.where(kv_idx < token_type_ids.shape[1], kv_idx, 0) + seq_length = group_ids.shape[-1] - token_type_ids_at_q_idx = token_type_ids[batch_idx, safe_q_idx] - token_type_ids_at_q_idx = torch.where(q_idx < token_type_ids.shape[1], token_type_ids_at_q_idx, 0) + # clamp indices because with static cache they can go beyond `group_ids.shape[-1]` + q_idx_clamped = q_idx.clamp(max=seq_length - 1) + kv_idx_clamped = kv_idx.clamp(max=seq_length - 1) - token_type_ids_at_kv_idx = token_type_ids[batch_idx, safe_kv_idx] - token_type_ids_at_kv_idx = torch.where(kv_idx < token_type_ids.shape[1], token_type_ids_at_kv_idx, 0) - - image_group_ids_at_q_idx = image_group_ids[batch_idx, safe_q_idx] - image_group_ids_at_q_idx = torch.where(q_idx < image_group_ids.shape[1], image_group_ids_at_q_idx, -1) - - image_group_ids_at_kv_idx = image_group_ids[batch_idx, safe_kv_idx] - image_group_ids_at_kv_idx = torch.where(kv_idx < image_group_ids.shape[1], image_group_ids_at_kv_idx, -1) - - is_image_block = (token_type_ids_at_q_idx == 1) & (token_type_ids_at_kv_idx == 1) - same_image_block = image_group_ids_at_q_idx == image_group_ids_at_kv_idx - - # This is bidirectional attention whenever we are dealing with image tokens - return is_image_block & same_image_block + # Unmask if the q and kv come from same group which is not -1 (i.e. non-text) + q_group = group_ids[batch_idx, q_idx_clamped] + kv_group = group_ids[batch_idx, kv_idx_clamped] + q_group = torch.where(q_idx < seq_length, q_group, -1) + kv_group = torch.where(kv_idx < seq_length, kv_group, -1) + return (q_group == kv_group) & (q_group >= 0) return inner_mask @@ -204,11 +191,9 @@ def create_causal_mask_mapping( is_image = (token_type_ids == 1).to(inputs_embeds.device) is_previous_image = nn.functional.pad(is_image, (1, 0), value=0)[:, :-1] new_image_start = is_image & ~is_previous_image - image_group_ids = torch.cumsum(new_image_start.int(), dim=1) - 1 - image_group_ids = torch.where(is_image, image_group_ids, torch.full_like(token_type_ids, -1)) - mask_kwargs["or_mask_function"] = token_type_ids_mask_function( - token_type_ids.to(inputs_embeds.device), image_group_ids - ) + group_ids = torch.cumsum(new_image_start.int(), dim=1) - 1 + group_ids = torch.where(is_image, group_ids, torch.full_like(token_type_ids, -1)) + mask_kwargs["or_mask_function"] = token_type_ids_mask_function(group_ids) return create_masks_for_generate(**mask_kwargs) @@ -400,12 +385,6 @@ def __init__(self, config): self.custom_text_proj = nn.Linear(self.config.text_config.hidden_size, self.embedding_dim) self.post_init() - def get_input_embeddings(self): - return self.model.get_input_embeddings() - - def set_input_embeddings(self, value): - self.model.set_input_embeddings(value) - @auto_docstring def get_image_features(self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]): return self.model.get_image_features(pixel_values, **kwargs) diff --git a/examples/modular-transformers/modeling_roberta.py b/examples/modular-transformers/modeling_roberta.py index 7ae436e70351..5163c704a922 100644 --- a/examples/modular-transformers/modeling_roberta.py +++ b/examples/modular-transformers/modeling_roberta.py @@ -217,14 +217,12 @@ def forward( **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor]: # determine input shapes - bsz, tgt_len = hidden_states.shape[:-1] - src_len = encoder_hidden_states.shape[1] + input_shape = hidden_states.shape[:-1] - q_input_shape = (bsz, tgt_len, -1, self.attention_head_size) - kv_input_shape = (bsz, src_len, -1, self.attention_head_size) + hidden_shape = (*input_shape, -1, self.attention_head_size) # get query proj - query_layer = self.query(hidden_states).view(*q_input_shape).transpose(1, 2) + query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2) is_updated = past_key_values.is_updated.get(self.layer_idx) if past_key_values is not None else False if past_key_values is not None and is_updated: @@ -232,8 +230,9 @@ def forward( key_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].keys value_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].values else: - key_layer = self.key(encoder_hidden_states).view(*kv_input_shape).transpose(1, 2) - value_layer = self.value(encoder_hidden_states).view(*kv_input_shape).transpose(1, 2) + kv_shape = (*encoder_hidden_states.shape[:-1], -1, self.attention_head_size) + key_layer = self.key(encoder_hidden_states).view(kv_shape).transpose(1, 2) + value_layer = self.value(encoder_hidden_states).view(kv_shape).transpose(1, 2) if past_key_values is not None: # save all states to the cache @@ -257,7 +256,7 @@ def forward( scaling=self.scaling, **kwargs, ) - attn_output = attn_output.reshape(bsz, tgt_len, -1).contiguous() + attn_output = attn_output.reshape(*input_shape, -1).contiguous() return attn_output, attn_weights diff --git a/examples/modular-transformers/modular_multimodal2.py b/examples/modular-transformers/modular_multimodal2.py index 81751d0815da..b11f8d1f294d 100644 --- a/examples/modular-transformers/modular_multimodal2.py +++ b/examples/modular-transformers/modular_multimodal2.py @@ -18,7 +18,6 @@ class Multimodal2VisionModel(CLIPVisionModel): CLIPEncoderLayer, CLIPPreTrainedModel, CLIPVisionModel, - CLIPVisionTransformer, ) @@ -54,17 +53,10 @@ def _init_weights(self, module): pass -# Finally here the `Vision` part was correct in CLIP, but we still need to tell it that the encoder and attn arg should -# use it as well -class Multimodal2VisionTransformer(CLIPVisionTransformer, Multimodal2VisionPreTrainedModel): - _no_split_modules = ["CLIPEncoderLayer"] +# `CLIPVisionModel` inherits from `CLIPPreTrainedModel`. We need to add the 2nd base here to add the `Vision` part +class Multimodal2VisionModel(CLIPVisionModel, Multimodal2VisionPreTrainedModel): + _no_split_modules = ["Multimodal2VisionEncoderLayer"] def __init__(self, config): super().__init__(config) self.encoder = Multimodal2VisionEncoder(config) - - -# Here the only arg `self.vision_model = CLIPVisionTransformer(config)` in CLIPVisionModel already has the "Vision" part, so -# no need to overwrite it, it will look for `Multimodal2VisionTransformer` which has already being redefined above -class Multimodal2VisionModel(CLIPVisionModel, Multimodal2VisionPreTrainedModel): - _no_split_modules = ["Multimodal2VisionEncoderLayer"] diff --git a/setup.py b/setup.py index 0fa835d5fb4a..d5daf2875bf8 100644 --- a/setup.py +++ b/setup.py @@ -124,7 +124,7 @@ "rjieba", "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1", "ruff==0.14.10", - "transformers-mlinter @ git+https://github.com/huggingface/transformers-mlinter@b9d319ce264c106f97a959d926ef42bc3c0ea4d1", + "transformers-mlinter==0.1.0", "ty==0.0.20", # `sacrebleu` not used in `transformers`. However, it is needed in several tests, when a test calls # `evaluate.load("sacrebleu")`. This metric is used in the examples that we use to test the `Trainer` with, in the @@ -295,7 +295,7 @@ def finalize_options(self): pass def run(self): - if SUPPORTED_PYTHON_VERSIONS[0] >= PYTHON_MINOR_VERSION: + if SUPPORTED_PYTHON_VERSIONS[0] > PYTHON_MINOR_VERSION: print( f"Table updated only when running 3.{SUPPORTED_PYTHON_VERSIONS[0]}.x, detected version is {sys.version}." ) diff --git a/src/transformers/cli/serving/chat_completion.py b/src/transformers/cli/serving/chat_completion.py index e39765020bdb..161a25a02f41 100644 --- a/src/transformers/cli/serving/chat_completion.py +++ b/src/transformers/cli/serving/chat_completion.py @@ -40,9 +40,9 @@ BaseGenerateManager, BaseHandler, Modality, - ToolCallParser, _StreamError, - detect_tool_format, + get_tool_call_config, + parse_tool_calls, ) @@ -140,11 +140,7 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse if use_cb: gen_manager.init_cb(model, gen_config) - # Detect tool support for the loaded model - # TODO: after tool_call start token, use constrained generation to: - # 1. force generation to pick from the available tool names - # 2. force generation to produce valid JSON matching the tool's parameter schema - tool_format = detect_tool_format(model) if body.get("tools") else None + tool_config = get_tool_call_config(processor, model) if body.get("tools") else None streaming = body.get("stream") if streaming: @@ -156,7 +152,7 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse inputs, gen_config, gen_manager=gen_manager, - tool_format=tool_format, + tool_config=tool_config, ) else: return await self._non_streaming( @@ -167,7 +163,7 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse inputs, gen_config, gen_manager=gen_manager, - tool_format=tool_format, + tool_config=tool_config, ) # ----- streaming ----- @@ -181,17 +177,22 @@ def _streaming( inputs: dict, gen_config: "GenerationConfig", gen_manager: BaseGenerateManager, - tool_format: dict | None = None, + tool_config: dict | None = None, ) -> StreamingResponse: """Stream tokens as SSE via DirectStreamer.""" - queue, streamer = gen_manager.generate_streaming(model, processor, inputs, gen_config, request_id=request_id) + queue, streamer = gen_manager.generate_streaming( + model, + processor, + inputs, + gen_config, + request_id=request_id, + tool_config=tool_config, + ) input_ids = inputs["input_ids"] # CB returns plain lists, regular path returns tensors input_len = len(input_ids) if isinstance(input_ids, list) else input_ids.shape[-1] - parser = ToolCallParser(tool_format) if tool_format else None async def sse_gen() -> AsyncGenerator[str, None]: - has_tool_calls = False try: yield self._build_chunk_sse(request_id, role="assistant", model=model_id) @@ -215,28 +216,32 @@ async def sse_gen() -> AsyncGenerator[str, None]: yield "".join(sse_parts) return - # Tool call parsing: None = normal text, CONSUMED = buffering, else = tool call dict - chunk_kwargs = {"content": text} - if parser is not None and (result := parser.feed(text)) is not None: - if result is ToolCallParser.CONSUMED: - continue - has_tool_calls = True - chunk_kwargs = { - "tool_calls": [ - ChoiceDeltaToolCall( - index=0, - type="function", - id=f"{request_id}_tool_call", - function={"name": result["name"], "arguments": result["arguments"]}, - ) - ] - } - - sse_parts.append(self._build_chunk_sse(request_id, model=model_id, **chunk_kwargs)) + sse_parts.append(self._build_chunk_sse(request_id, model=model_id, content=text)) if sse_parts: yield "".join(sse_parts) + # Tool calls are parsed after generation completes (not during streaming), + # because the full token sequence is needed for reliable parsing. + has_tool_calls = False + if tool_config: + parsed = parse_tool_calls(processor, streamer.generated_token_ids, tool_config["schema"]) + if parsed: + has_tool_calls = True + for i, tc in enumerate(parsed): + yield self._build_chunk_sse( + request_id, + model=model_id, + tool_calls=[ + ChoiceDeltaToolCall( + index=i, + type="function", + id=f"{request_id}_tool_call_{i}", + function={"name": tc["name"], "arguments": tc["arguments"]}, + ) + ], + ) + hit_max = gen_config.max_new_tokens is not None and streamer.total_tokens >= gen_config.max_new_tokens if has_tool_calls: finish_reason = "tool_calls" @@ -274,7 +279,7 @@ async def _non_streaming( inputs: dict, gen_config: "GenerationConfig", gen_manager: BaseGenerateManager, - tool_format: dict | None = None, + tool_config: dict | None = None, ) -> JSONResponse: """Run generation and return a JSONResponse.""" content, input_len, generated_ids = await gen_manager.generate_non_streaming( @@ -289,18 +294,17 @@ async def _non_streaming( total_tokens=input_len + completion_tokens, ) - # Parse tool calls from the generated text tool_calls = None - if tool_format is not None: - parsed = ToolCallParser.parse(content, tool_format) - if parsed is not None: + if tool_config is not None: + parsed = parse_tool_calls(processor, generated_ids, tool_config["schema"]) + if parsed: tool_calls = [ ChatCompletionMessageToolCall( - id=f"{request_id}_tool_call", + id=f"{request_id}_tool_call_{i}", type="function", function={"name": tc["name"], "arguments": tc["arguments"]}, ) - for tc in parsed + for i, tc in enumerate(parsed) ] if tool_calls is not None: diff --git a/src/transformers/cli/serving/response.py b/src/transformers/cli/serving/response.py index 2ab2d7291d04..4d29dfd1d6a2 100644 --- a/src/transformers/cli/serving/response.py +++ b/src/transformers/cli/serving/response.py @@ -56,9 +56,9 @@ BaseGenerateManager, BaseHandler, Modality, - ToolCallParser, _StreamError, - detect_tool_format, + get_tool_call_config, + parse_tool_calls, ) @@ -118,7 +118,7 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse # Two-step input conversion (chat completions skips step 1 since messages are already standard): # 1. Normalize Responses API input (string/list/dict + instructions) → standard messages list # 2. Transform message content for the HF processor (VLM image handling, text joining, etc.) - messages = self._input_to_messages(body) + messages = self._normalize_input(body) processor_inputs = self.get_processor_inputs_from_messages(messages, modality) has_video = any( @@ -131,10 +131,12 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse chat_template_kwargs = {} if has_video: chat_template_kwargs["num_frames"] = 32 + # updates the flat tool structure to the one expected by the `apply_chat_template` method. + tools = self._normalize_tools(body.get("tools")) inputs = processor.apply_chat_template( processor_inputs, add_generation_prompt=True, - tools=body.get("tools"), + tools=tools, return_tensors=None if use_cb else "pt", return_dict=True, tokenize=True, @@ -148,7 +150,7 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse # TODO: remove when CB supports per-request generation config if use_cb: gen_manager.init_cb(model, gen_config) - tool_format = detect_tool_format(model) if body.get("tools") else None + tool_config = get_tool_call_config(processor, model) if body.get("tools") else None streaming = body.get("stream", True) if streaming: @@ -161,7 +163,7 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse inputs, gen_config, gen_manager=gen_manager, - tool_format=tool_format, + tool_config=tool_config, ) else: return await self._non_streaming( @@ -173,58 +175,111 @@ async def handle_request(self, body: dict, request_id: str) -> StreamingResponse inputs, gen_config, gen_manager=gen_manager, - tool_format=tool_format, + tool_config=tool_config, ) # ----- input conversion ----- @staticmethod - def _input_to_messages(body: dict) -> list[dict]: - """Convert the Responses API ``input`` field to a list of chat messages. + def _normalize_tools(tools: list[dict] | None) -> list[dict] | None: + """Normalize Responses API tool definitions for ``apply_chat_template``. - The Responses API ``input`` field accepts several formats. This method normalizes - all of them into a standard list of messages with ``role`` and ``content`` keys. - - Supported input formats: - 1. **String**: ``input="Hello"`` → ``[{"role": "user", "content": "Hello"}]`` - 2. **Flat content list** (Responses API native, no ``role`` key): - ``input=[{"type": "input_text", "text": "..."}, {"type": "input_image", ...}]`` - → wrapped as a single user message. - 3. **Messages list** (multi-turn, with ``role`` keys): - ``input=[{"role": "user", "content": [...]}, {"role": "assistant", ...}]`` - → passed through as-is. - - If ``instructions`` is provided, it is prepended as a system message (or replaces - an existing one). - - Args: - body (`dict`): The raw request body containing ``input`` and optionally ``instructions``. + The Responses API uses a flat format: ``{"type": "function", "name": ..., "parameters": ...}`` + while ``apply_chat_template`` expects a nested format: + ``{"type": "function", "function": {"name": ..., "parameters": ...}}``. + Already-nested tools are passed through unchanged. + """ + if not tools: + return tools + return [ + {"type": "function", "function": {k: v for k, v in t.items() if k != "type"}} if "function" not in t else t + for t in tools + ] - Returns: - `list[dict]`: Standard chat messages with ``role`` and ``content`` keys. + @staticmethod + def _normalize_input(body: dict) -> list[dict]: + """Normalize the Responses API ``input`` field into chat messages. + + The Responses API accepts multiple input formats. This method converts them + into a structure close to what ``apply_chat_template`` expects (messages with + ``role``, ``content``, ``tool_calls``, ``tool_call_id``). Further processing + is done by ``get_processor_inputs_from_messages``. + + NOTE: if this conversion logic grows too complex, consider having separate + ``get_processor_inputs_from_messages`` implementations for chat completions + and the Responses API instead of funneling both through the same path. + + Formats handled: + - **String** → single user message. + - **Flat content list** (``input_text``, ``input_image``, no ``role``) → user message. + - **Multi-turn list** — messages and tool call items (``function_call``, + ``function_call_output``) from a previous response, converted via + :meth:`_normalize_response_items`. + + If ``instructions`` is present, it is prepended as a system message. """ inp = body["input"] instructions = body.get("instructions") if isinstance(inp, str): - messages = [{"role": "system", "content": instructions}] if instructions else [] - messages.append({"role": "user", "content": inp}) + messages = [{"role": "user", "content": inp}] elif isinstance(inp, list): - # Flat content list (no "role" key) — wrap as a single user message - if inp and "type" in inp[0] and "role" not in inp[0]: - messages = [{"role": "system", "content": instructions}] if instructions else [] - messages.append({"role": "user", "content": inp}) - elif instructions: - if inp[0]["role"] != "system": - messages = [{"role": "system", "content": instructions}, *inp] - else: - messages = list(inp) - messages[0]["content"] = instructions + if inp and "role" not in inp[0]: + # Flat content list (single-turn, e.g. input_text/input_image) + messages = [{"role": "user", "content": inp}] else: - messages = inp + messages = ResponseHandler._normalize_response_items(inp) else: raise HTTPException(status_code=422, detail="'input' must be a string or list") + # Prepend instructions as a system message + if instructions: + if messages and messages[0]["role"] == "system": + messages[0]["content"] = instructions + else: + messages.insert(0, {"role": "system", "content": instructions}) + + return messages + + @staticmethod + def _normalize_response_items(items: list[dict]) -> list[dict]: + """Convert a list of Responses API items into chat messages. + + Input items may be a mix of: + - Messages (``EasyInputMessageParam`` with ``role``, or ``type: "message"``). + - ``function_call`` — merged as ``tool_calls`` onto the preceding assistant message. + - ``function_call_output`` — converted to ``role: "tool"`` messages. + """ + messages = [] + + for item in items: + item_type = item.get("type") + + if "role" in item: + messages.append({"role": item["role"], "content": item.get("content", "")}) + + elif item_type == "function_call": + tc = { + "id": item["call_id"], + "function": {"name": item["name"], "arguments": item["arguments"]}, + } + if messages and messages[-1]["role"] == "assistant": + messages[-1].setdefault("tool_calls", []).append(tc) + else: + messages.append({"role": "assistant", "tool_calls": [tc]}) + + elif item_type == "function_call_output": + messages.append( + { + "role": "tool", + "tool_call_id": item["call_id"], + "content": item["output"], + } + ) + + else: + raise HTTPException(status_code=422, detail=f"Unsupported input item type: {item_type!r}") + return messages # ----- streaming ----- @@ -239,14 +294,20 @@ def _streaming( inputs: dict, gen_config: "GenerationConfig", gen_manager: BaseGenerateManager, - tool_format: dict | None = None, + tool_config: dict | None = None, ) -> StreamingResponse: """Generate a streaming Responses API reply (SSE) using DirectStreamer.""" - queue, streamer = gen_manager.generate_streaming(model, processor, inputs, gen_config, request_id=request_id) + queue, streamer = gen_manager.generate_streaming( + model, + processor, + inputs, + gen_config, + request_id=request_id, + tool_config=tool_config, + ) input_ids = inputs["input_ids"] # CB returns plain lists, regular path returns tensors input_len = len(input_ids) if isinstance(input_ids, list) else input_ids.shape[-1] - parser = ToolCallParser(tool_format) if tool_format else None seq = 0 output_index = 0 @@ -362,59 +423,6 @@ async def event_stream() -> AsyncGenerator[str, None]: yield "".join(sse_parts) return - # Tool call parsing - if parser is not None and (result := parser.feed(text)) is not None: - if result is not ToolCallParser.CONSUMED: - tc_id = f"{request_id}_tool_call" - name = result["name"] - arguments = result["arguments"] - tc_item = ResponseFunctionToolCall( - id=tc_id, - call_id=tc_id, - type="function_call", - name=name, - arguments=arguments, - status="completed", - ) - tool_calls.append(tc_item) - output_index += 1 - sse_parts.append( - self.chunk_to_sse( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=seq, - output_index=output_index, - item=tc_item, - ) - ) - ) - seq += 1 - sse_parts.append( - self.chunk_to_sse( - ResponseFunctionCallArgumentsDoneEvent( - type="response.function_call_arguments.done", - sequence_number=seq, - item_id=tc_id, - output_index=output_index, - arguments=arguments, - name=name, - ) - ) - ) - seq += 1 - sse_parts.append( - self.chunk_to_sse( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=seq, - output_index=output_index, - item=tc_item, - ) - ) - ) - seq += 1 - continue - full_text += text sse_parts.append( self.chunk_to_sse( @@ -434,7 +442,54 @@ async def event_stream() -> AsyncGenerator[str, None]: if sse_parts: yield "".join(sse_parts) - # 5. Close text output + # 5. Tool calls are parsed after generation completes (not during streaming), + # because the full token sequence is needed for reliable parsing. + if tool_config: + parsed = parse_tool_calls(processor, streamer.generated_token_ids, tool_config["schema"]) + if parsed: + for i, tc in enumerate(parsed): + tc_id = f"{request_id}_tool_call_{i}" + tc_item = ResponseFunctionToolCall( + id=tc_id, + call_id=tc_id, + type="function_call", + name=tc["name"], + arguments=tc["arguments"], + status="completed", + ) + tool_calls.append(tc_item) + output_index += 1 + yield self.chunk_to_sse( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=seq, + output_index=output_index, + item=tc_item, + ) + ) + seq += 1 + yield self.chunk_to_sse( + ResponseFunctionCallArgumentsDoneEvent( + type="response.function_call_arguments.done", + sequence_number=seq, + item_id=tc_id, + output_index=output_index, + arguments=tc["arguments"], + name=tc["name"], + ) + ) + seq += 1 + yield self.chunk_to_sse( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=seq, + output_index=output_index, + item=tc_item, + ) + ) + seq += 1 + + # 6. Close text output output_text_part = ResponseOutputText(type="output_text", text=full_text, annotations=[]) yield self.chunk_to_sse( ResponseTextDoneEvent( @@ -478,7 +533,7 @@ async def event_stream() -> AsyncGenerator[str, None]: ) seq += 1 - # 6. Completed + # 7. Completed all_output = [msg_item] + list(tool_calls) usage = compute_usage(input_len, streamer.total_tokens) yield self.chunk_to_sse( @@ -509,7 +564,7 @@ async def _non_streaming( inputs: dict, gen_config: "GenerationConfig", gen_manager: BaseGenerateManager, - tool_format: dict | None = None, + tool_config: dict | None = None, ) -> JSONResponse: """Generate a non-streaming Responses API reply (single JSON).""" full_text, input_len, generated_ids = await gen_manager.generate_non_streaming( @@ -527,12 +582,11 @@ async def _non_streaming( ) ] - # Parse tool calls from the generated text - if tool_format is not None: - parsed_calls = ToolCallParser.parse(full_text, tool_format) - if parsed_calls is not None: - for i, tc in enumerate(parsed_calls): - tc_id = f"{request_id}_tool_call" + if tool_config is not None: + parsed = parse_tool_calls(processor, generated_ids, tool_config["schema"]) + if parsed: + for i, tc in enumerate(parsed): + tc_id = f"{request_id}_tool_call_{i}" output_items.append( ResponseFunctionToolCall( id=tc_id, diff --git a/src/transformers/cli/serving/utils.py b/src/transformers/cli/serving/utils.py index 3ef55d5f0041..d786a828fc28 100644 --- a/src/transformers/cli/serving/utils.py +++ b/src/transformers/cli/serving/utils.py @@ -73,132 +73,87 @@ class _GenerationCancelled(Exception): """Raised inside ``DirectStreamer.put()`` to abort ``model.generate()``.""" -# Model-specific tokens that mark the start/end of a tool call block. -# TODO: extract these from the chat template at runtime instead of hardcoding. -# Qwen/Hermes use /, Mistral uses [TOOL_CALLS], etc. -# The markers are defined in each model's Jinja chat template. -_TOOL_CALL_TOKENS = { +# Fallback tool call configs for models that don't declare stc_token/etc_token/response_schema +# on their tokenizer. +# Keys are matched via substring against model_type (e.g. "qwen" matches "qwen2", "qwen3_vl", etc.). +# If a model family changes its tool call format, split into separate keys (e.g. "qwen2", "qwen3"). +_TOOL_CALL_FALLBACKS = { "qwen": { - "start": "", - "end": "", + "stc": "", + "etc": "", + "schema": { + "x-regex-iterator": r"(.*?)", + "type": "array", + "items": {"type": "object", "x-parser": "json"}, + }, }, } -def detect_tool_format(model: "PreTrainedModel") -> dict | None: - """Return the tool call token format for a model, if supported. +def get_tool_call_config(processor, model: "PreTrainedModel") -> dict | None: + """Return tool call config for the model, or ``None`` if tool calls are not supported. - Args: - model (`PreTrainedModel`): The loaded model. - - Returns: - `dict | None`: A dict ``{"start": str, "end": str}`` with the model's tool call - delimiters, or ``None`` if the model family is not recognized. + Returns a dict with: + - ``schema`` (`dict`): Schema to pass to ``tokenizer.parse_response(block, schema)``. + - ``stc_id`` (`int`): Token ID of the start-of-tool-call delimiter. + - ``etc_id`` (`int`): Token ID of the end-of-tool-call delimiter. """ - architecture = model.config.architectures[0].lower() - for family in _TOOL_CALL_TOKENS: - if family in architecture: - return _TOOL_CALL_TOKENS[family] - return None - - -class ToolCallParser: - """Parses tool calls from model output. + tokenizer = getattr(processor, "tokenizer", processor) + stc = getattr(tokenizer, "stc_token", None) + etc = getattr(tokenizer, "etc_token", None) + response_schema = getattr(tokenizer, "response_schema", None) + + # Models with full tokenizer config (e.g. Gemma 4) + if stc and etc and response_schema: + schema = response_schema["properties"]["tool_calls"] + else: + # Fallback: known model families without full tokenizer config + fallback = next((v for k, v in _TOOL_CALL_FALLBACKS.items() if k in model.config.model_type), None) + if fallback is None: + return None + stc, etc, schema = fallback["stc"], fallback["etc"], fallback["schema"] - The model emits tool calls as structured text between start/end tokens - (e.g. ``{"name": "fn", "arguments": {...}}``). + stc_id = tokenizer.convert_tokens_to_ids(stc) + etc_id = tokenizer.convert_tokens_to_ids(etc) + return {"schema": schema, "stc_id": stc_id, "etc_id": etc_id} - **Streaming** (``feed``): buffers tokens between start/end markers, parses - the complete block when the end marker is seen, returns a ``ChoiceDeltaToolCall``. - **Non-streaming** (``parse``): extracts all tool call blocks from complete text. +def _normalize_tool_call(tool_call: dict) -> dict: + """Normalize a parsed tool call to ``{"name": str, "arguments": str}``. - Usage:: + Different models return different structures from ``parse_response``: + - Gemma: ``{"function": {"name": ..., "arguments": {...}}}`` (nested, arguments as dict) + - Qwen: ``{"name": ..., "arguments": {...}}`` (flat, arguments as dict) - parser = ToolCallParser(tool_format={"start": ..., "end": ...}) - for text_chunk in streamer: - result = parser.feed(text_chunk) - if result is None: - # Normal text — emit as content - elif result is ToolCallParser.CONSUMED: - # Buffering — skip - else: - # result is a ChoiceDeltaToolCall — emit it + The OpenAI API expects ``arguments`` as a JSON **string**, so we ``json.dumps`` it. """ + function = tool_call.get("function", tool_call) + arguments = function.get("arguments", {}) + return { + "name": function["name"], + "arguments": json.dumps(arguments) if not isinstance(arguments, str) else arguments, + } - def __init__(self, tool_format: dict): - self._tokens = tool_format - self._inside = False - self._buffer = "" - - # Sentinel: token was consumed by the parser but produced no output. - CONSUMED = object() - - def feed(self, text: str) -> object | dict | None: - """Feed a text chunk (streaming). - Returns: - - ``None`` — normal text, not a tool token. Emit as content. - - ``CONSUMED`` — token consumed internally (buffering/markers). Skip. - - A ``ChoiceDeltaToolCall`` — emit as a tool call delta. - """ - if text.strip() == self._tokens["start"]: - self._inside = True - self._buffer = "" - return self.CONSUMED +def parse_tool_calls(processor, generated_ids, schema: dict) -> list[dict] | None: + """Parse tool calls from generated token IDs using ``tokenizer.parse_response``. - if text.strip() == self._tokens["end"]: - self._inside = False - block = self._buffer.strip() - self._buffer = "" - return self._parse_block(block) or self.CONSUMED - - if self._inside: - self._buffer += text - return self.CONSUMED + Args: + processor: The processor or tokenizer. + generated_ids: Token IDs from generation. Passed directly to ``parse_response`` + which decodes them internally, preserving special tokens that + ``skip_special_tokens=True`` would strip (e.g. Gemma's ``<|tool_call>``). + schema: The tool call schema (from ``response_schema`` or ``_TOOL_CALL_FALLBACKS``). + Returns a list of ``{"name": str, "arguments": str}`` dicts, or ``None`` if none found. + """ + parsed = processor.parse_response(generated_ids, schema) + if not parsed: return None - - @staticmethod - def _extract_name_and_args(block: str) -> tuple[str, str] | None: - """Extract (name, arguments_json) from a tool call block, or None if invalid.""" - if not block: - return None - parsed = json.loads(block) - name = parsed.get("name") - if name is None: - return None - arguments = parsed.get("arguments", {}) - return name, json.dumps(arguments) - - @staticmethod - def parse(text: str, tool_format: dict) -> list[dict] | None: - """Parse tool calls from complete text. - - Returns a list of ``{"name": str, "arguments": str}`` dicts, or ``None`` if none found. - """ - start, end = tool_format["start"], tool_format["end"] - tool_calls = [] - pos = 0 - while True: - s = text.find(start, pos) - if s < 0: - break - e = text.find(end, s + len(start)) - if e < 0: - break - result = ToolCallParser._extract_name_and_args(text[s + len(start) : e].strip()) - if result is not None: - tool_calls.append({"name": result[0], "arguments": result[1]}) - pos = e + len(end) - return tool_calls if tool_calls else None - - def _parse_block(self, block: str) -> dict | None: - """Parse a buffered tool call block. Returns ``{"name": str, "arguments": str}`` or None.""" - result = self._extract_name_and_args(block) - if result is None: - return None - return {"name": result[0], "arguments": result[1]} + if not isinstance(parsed, list): + parsed = [parsed] + tool_calls = [_normalize_tool_call(tool_call) for tool_call in parsed] + return tool_calls if tool_calls else None class DownloadAggregator: @@ -330,6 +285,7 @@ def __init__( loop: asyncio.AbstractEventLoop, queue: asyncio.Queue, skip_special_tokens: bool = True, + tool_config: dict | None = None, ): """ Args: @@ -338,6 +294,9 @@ def __init__( queue (`asyncio.Queue`): The queue that receives decoded text chunks. skip_special_tokens (`bool`, *optional*, defaults to `True`): Whether to strip special tokens during decoding. + tool_config (`dict`, *optional*): Tool call config from ``get_tool_call_config``. + When set, tokens between stc/etc delimiters (inclusive) are suppressed + from the queue so tool call markup is never streamed to the client. """ from tokenizers.decoders import DecodeStream @@ -345,9 +304,13 @@ def __init__( self._loop = loop self._queue = queue self._decode_stream = DecodeStream([], skip_special_tokens) + self._stc_id = tool_config["stc_id"] if tool_config else None + self._etc_id = tool_config["etc_id"] if tool_config else None + self._inside_tool_call = False self._first = True self._cancelled = threading.Event() self.total_tokens = 0 + self.generated_token_ids: list[int] = [] def put(self, value: "torch.Tensor") -> None: """Called by ``model.generate()`` after each decode step with new token(s).""" @@ -359,8 +322,15 @@ def put(self, value: "torch.Tensor") -> None: return for token_id in value.tolist(): self.total_tokens += 1 + self.generated_token_ids.append(token_id) + + if token_id == self._stc_id: + self._inside_tool_call = True + elif token_id == self._etc_id: + self._inside_tool_call = False + text = self._decode_stream.step(self._tokenizer, token_id) - if text is not None: + if text is not None and not self._inside_tool_call and token_id != self._etc_id: self._loop.call_soon_threadsafe(self._queue.put_nowait, text) def end(self) -> None: @@ -388,6 +358,7 @@ def __init__( tokenizer: "tokenizers.Tokenizer", loop: asyncio.AbstractEventLoop, queue: asyncio.Queue, + tool_config: dict | None = None, ): """ Args: @@ -396,6 +367,7 @@ def __init__( tokenizer: The Rust tokenizer (``tokenizer._tokenizer``). loop (`asyncio.AbstractEventLoop`): The event loop to push decoded text to. queue (`asyncio.Queue`): The queue that receives decoded text chunks. + tool_config (`dict`, *optional*): Tool call config (see ``DirectStreamer``). """ from tokenizers.decoders import DecodeStream @@ -405,8 +377,12 @@ def __init__( self._queue = queue self._tokenizer = tokenizer self._decode_stream = DecodeStream([], True) + self._stc_id = tool_config["stc_id"] if tool_config else None + self._etc_id = tool_config["etc_id"] if tool_config else None + self._inside_tool_call = False self._prev_len = 0 self.total_tokens = 0 + self.generated_token_ids: list[int] = [] def put(self, output: "GenerationOutput") -> None: """Decode new tokens from a CB ``GenerationOutput`` and push text to the queue.""" @@ -414,8 +390,15 @@ def put(self, output: "GenerationOutput") -> None: self._prev_len = len(output.generated_tokens) for token_id in new_tokens: self.total_tokens += 1 + self.generated_token_ids.append(token_id) + + if token_id == self._stc_id: + self._inside_tool_call = True + elif token_id == self._etc_id: + self._inside_tool_call = False + text = self._decode_stream.step(self._tokenizer, token_id) - if text is not None: + if text is not None and not self._inside_tool_call and token_id != self._etc_id: self._queue.put_nowait(text) def end(self) -> None: @@ -502,6 +485,7 @@ def generate_streaming( inputs: dict, gen_config: "GenerationConfig", request_id: str, + tool_config: dict | None = None, ) -> tuple[asyncio.Queue, "DirectStreamer | CBStreamer"]: """Start streaming generation. @@ -511,6 +495,8 @@ def generate_streaming( inputs (`dict`): Tokenized inputs (tensors for sequential, lists for CB). gen_config (`GenerationConfig`): Generation parameters. request_id (`str`): Unique request identifier. + tool_config (`dict`, *optional*): Tool call config from ``get_tool_call_config``. + When set, tool call tokens (between stc/etc) are suppressed from output. Returns: `tuple[asyncio.Queue, DirectStreamer | CBStreamer]`: A ``(queue, streamer)`` pair @@ -558,13 +544,14 @@ def generate_streaming( inputs: dict, gen_config: "GenerationConfig", request_id: str, + tool_config: dict | None = None, ) -> tuple[asyncio.Queue, DirectStreamer]: """Start streaming generation via ``model.generate()`` on the inference thread.""" loop = asyncio.get_running_loop() queue: asyncio.Queue = asyncio.Queue() # ProcessorMixin exposes the fast tokenizer as .tokenizer; PreTrainedTokenizerFast is already one. rust_tokenizer = getattr(processor, "tokenizer", processor)._tokenizer # type: ignore[union-attr] - streamer = DirectStreamer(rust_tokenizer, loop, queue, skip_special_tokens=True) + streamer = DirectStreamer(rust_tokenizer, loop, queue, tool_config=tool_config) gen_kwargs = {**inputs, "streamer": streamer, "generation_config": gen_config, "tokenizer": processor} if hasattr(model, "has_talker"): gen_kwargs["generation_mode"] = "text" @@ -655,6 +642,7 @@ def generate_streaming( inputs: dict, gen_config: "GenerationConfig", request_id: str, + tool_config: dict | None = None, ) -> tuple[asyncio.Queue, CBStreamer]: """Start streaming CB generation. Registers a per-request output handler.""" cb = self._cb @@ -674,7 +662,7 @@ def generate_streaming( ) # ProcessorMixin exposes the fast tokenizer as .tokenizer; PreTrainedTokenizerFast is already one. rust_tokenizer = getattr(processor, "tokenizer", processor)._tokenizer # type: ignore[union-attr] - streamer = CBStreamer(self._cb, request_id, rust_tokenizer, loop, text_queue) + streamer = CBStreamer(self._cb, request_id, rust_tokenizer, loop, text_queue, tool_config=tool_config) # Register a direct callback: the dispatcher calls this on the event loop with each GenerationOutput. # This decodes tokens and pushes text straight to the SSE text_queue @@ -867,7 +855,15 @@ def _resolve_model(self, body: dict) -> tuple[str, "PreTrainedModel", "Processor Returns ``(model_id, model, processor)``. """ + from fastapi import HTTPException + if self.model_manager.force_model is not None: + requested = body.get("model") + if requested is not None and requested != self.model_manager.force_model: + raise HTTPException( + status_code=400, + detail=(f"Server is pinned to '{self.model_manager.force_model}'; requested '{requested}'."), + ) body["model"] = self.model_manager.force_model model_id = self.model_manager.process_model_name(body["model"]) @@ -951,14 +947,16 @@ def get_processor_inputs_from_messages(messages: list[dict], modality: Modality) if "tool_call_id" in message: parsed["tool_call_id"] = message["tool_call_id"] - raw_content = message.get("content", []) + # When tool_calls are present, ignore content — it's either empty or contains + # raw tool call markup that would confuse the chat template if rendered. + raw_content = [] if "tool_calls" in message else (message.get("content") or []) if isinstance(raw_content, str): raw_content = [{"type": "text", "text": raw_content}] for content in raw_content: content_type = content["type"] # Text: chat completions ("text") and Responses API ("input_text") - if content_type in ("text", "input_text"): + if content_type in ("text", "input_text", "output_text"): parsed["content"].append({"type": "text", "text": content["text"]}) # Image: chat completions ("image_url") and Responses API ("input_image") elif content_type in ("image_url", "input_image") and modality in (Modality.VLM, Modality.MULTIMODAL): diff --git a/src/transformers/conversion_mapping.py b/src/transformers/conversion_mapping.py index 77002751fa23..39965f6e8406 100755 --- a/src/transformers/conversion_mapping.py +++ b/src/transformers/conversion_mapping.py @@ -75,7 +75,6 @@ "qwen2_5_vl": "qwen2_vl", "sam3_tracker_video": "sam3_tracker", "pp_chart2table": "llava", - "gemma3n_text": "qwen3_5_text", "qwen3_5_moe_text": "qwen3_5_text", "altclip_vision_model": "clip_vision_model", "chinese_clip_vision_model": "clip_vision_model", @@ -152,19 +151,10 @@ def _build_checkpoint_conversion_mapping(): WeightRenaming(source_patterns=r"^visual", target_patterns="model.visual"), ], "colqwen2": [ - WeightRenaming(source_patterns=r"vlm.model", target_patterns="vlm"), + PrefixChange(prefix_to_remove="model", model_prefix="vlm"), WeightRenaming(source_patterns=r"vlm(?!\.(language_model|visual))", target_patterns="vlm.language_model"), ], - "timm_wrapper": [ - # Simply add the prefix `timm_model`. Similar to `base_model_prefix` but also removes prefix - # when saving. TODO: Would be probably much cleaner with a `add_prefix` argument in WeightRenaming - # Note: we don't add `timm_model` when it is part of a bigger VLM, because they already have `timm_model` - # saved in state dict keys. Thus the look behind check. Should be fixed by proper `add_prefix`! - WeightRenaming( - source_patterns=r"^(?!(?:model\.|backbone\.|tower\.))(.+)$", - target_patterns=r"timm_model.\1", - ) - ], + "timm_wrapper": [PrefixChange(prefix_to_add="timm_model")], "pi0": [ WeightRenaming(source_patterns=r"state_proj", target_patterns="embed_action_time.state_proj"), WeightRenaming(source_patterns=r"action_in_proj", target_patterns="embed_action_time.action_in_proj"), @@ -202,13 +192,7 @@ def _build_checkpoint_conversion_mapping(): WeightRenaming("attention_layer_norm", "input_layernorm"), WeightRenaming("feedforward_layer_norm", "post_attention_layernorm"), ], - "qwen3_5_text": [ - # Note: the lookbehind on the target is to avoid replacing bigger matches when the model is a submodel of - # the ForConditionalGeneration model - WeightRenaming( - source_patterns=r"^model.language_model.", target_patterns=r"^model.(?!(?:language_model.|visual.))" - ), - ], + "qwen3_5_text": [PrefixChange(prefix_to_remove="language_model", model_prefix="model")], "sam3_tracker": [ WeightRenaming( source_patterns=r"detector_model.vision_encoder.backbone.", target_patterns="vision_encoder.backbone." @@ -475,16 +459,6 @@ def _build_checkpoint_conversion_mapping(): operations=[MergeModulelist(dim=0)], ), ], - "legacy": [ - WeightRenaming( - source_patterns="LayerNorm.gamma", - target_patterns="LayerNorm.weight", - ), - WeightRenaming( - source_patterns="LayerNorm.beta", - target_patterns="LayerNorm.bias", - ), - ], "nomic_bert": [ WeightRenaming(r"encoder.layers", r"layers"), WeightRenaming(r"emb_ln", r"embeddings.LayerNorm"), @@ -523,6 +497,38 @@ def _build_checkpoint_conversion_mapping(): WeightRenaming(source_patterns="norm1", target_patterns="post_attention_layernorm"), WeightRenaming(source_patterns="norm2", target_patterns="post_mlp_layernorm"), ], + "cohere_asr": [ + WeightRenaming(r"encoder\.pre_encode\.conv\.", r"encoder.subsampling.layers."), + WeightRenaming(r"encoder\.pre_encode\.out\.", r"encoder.subsampling.linear."), + WeightRenaming(r"transf_decoder\._embedding\.position_embedding\.pos_enc", r"decoder.pos_emb.weight"), + WeightRenaming(r"transf_decoder\._embedding\.token_embedding", r"decoder.embed_tokens"), + WeightRenaming(r"transf_decoder\._embedding\.layer_norm", r"decoder.embedding_layernorm"), + WeightRenaming(r"transf_decoder\._decoder\.final_layer_norm", r"decoder.norm"), + WeightRenaming(r"transf_decoder\._decoder\.layers", r"decoder.layers"), + WeightRenaming(r"encoder_decoder_proj\.", r"decoder.proj."), + WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_q", r"encoder.\1.self_attn.q_proj"), + WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_k", r"encoder.\1.self_attn.k_proj"), + WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_v", r"encoder.\1.self_attn.v_proj"), + WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_out", r"encoder.\1.self_attn.o_proj"), + WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_pos", r"encoder.\1.self_attn.relative_k_proj"), + WeightRenaming(r"encoder\.(.+)\.self_attn\.pos_bias_u", r"encoder.\1.self_attn.bias_u"), + WeightRenaming(r"encoder\.(.+)\.self_attn\.pos_bias_v", r"encoder.\1.self_attn.bias_v"), + WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.query_net", r"decoder.\1.self_attn.q_proj"), + WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.key_net", r"decoder.\1.self_attn.k_proj"), + WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.value_net", r"decoder.\1.self_attn.v_proj"), + WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.out_projection", r"decoder.\1.self_attn.o_proj"), + WeightRenaming(r"\.second_sub_layer\.query_net", r".encoder_attn.q_proj"), + WeightRenaming(r"\.second_sub_layer\.key_net", r".encoder_attn.k_proj"), + WeightRenaming(r"\.second_sub_layer\.value_net", r".encoder_attn.v_proj"), + WeightRenaming(r"\.second_sub_layer\.out_projection", r".encoder_attn.o_proj"), + WeightRenaming(r"\.third_sub_layer\.dense_in", r".mlp.fc1"), + WeightRenaming(r"\.third_sub_layer\.dense_out", r".mlp.fc2"), + WeightRenaming(r"\.layer_norm_1\.", r".input_layernorm."), + WeightRenaming(r"\.layer_norm_2\.", r".post_attention_layernorm."), + WeightRenaming(r"\.layer_norm_3\.", r".final_layernorm."), + WeightRenaming(r"\.conv\.batch_norm", r".conv.norm"), + WeightRenaming(r"log_softmax\.mlp\.layer0", r"proj_out"), + ], "qianfan_ocr": [ WeightRenaming(r"^vision_model\.", r"model\.vision_tower\."), WeightRenaming(r"encoder\.layers\.", r"layers\."), @@ -550,6 +556,16 @@ def _build_checkpoint_conversion_mapping(): WeightRenaming(r"^mlp1\.1\.", r"model\.multi_modal_projector\.linear_1\."), WeightRenaming(r"^mlp1\.3\.", r"model\.multi_modal_projector\.linear_2\."), ], + "legacy": [ + WeightRenaming( + source_patterns="LayerNorm.gamma", + target_patterns="LayerNorm.weight", + ), + WeightRenaming( + source_patterns="LayerNorm.beta", + target_patterns="LayerNorm.bias", + ), + ], } # The legacy mapping is added to the esm model here since the extra weight renaming do not apply to the esm model. mapping["esm"] += mapping["legacy"].copy() @@ -565,22 +581,11 @@ def _build_checkpoint_conversion_mapping(): ), ] - mapping["ernie4_5_moe"] = [ - WeightRenaming("mlp.moe_statics.e_score_correction_bias", "mlp.gate.moe_statics.e_score_correction_bias"), - WeightConverter( - source_patterns=[ - "mlp.experts.*.gate_proj.weight", - "mlp.experts.*.up_proj.weight", - ], - target_patterns="mlp.experts.gate_up_proj", - operations=[MergeModulelist(dim=0), Concatenate(dim=1)], - ), - WeightConverter( - source_patterns="mlp.experts.*.down_proj.weight", - target_patterns="mlp.experts.down_proj", - operations=[MergeModulelist(dim=0)], - ), + mapping["ernie4_5_moe"] = mapping["qwen2_moe"].copy() + mapping["ernie4_5_moe"] += [ + WeightRenaming("mlp.moe_statics.e_score_correction_bias", "mlp.gate.moe_statics.e_score_correction_bias") ] + mapping["minimax_m2"] = mapping["mixtral"].copy() mapping["minimax_m2"] += [ WeightRenaming(".block_sparse_moe.e_score_correction_bias", ".mlp.e_score_correction_bias"), @@ -588,55 +593,6 @@ def _build_checkpoint_conversion_mapping(): mapping["exaone_moe"] = mapping["qwen2_moe"].copy() mapping["exaone_moe"] += [WeightRenaming("mlp.e_score_correction_bias", "mlp.gate.e_score_correction_bias")] - mapping["solar_open"] = [ - WeightConverter( - source_patterns=[ - "mlp.experts.*.gate_proj.weight", - "mlp.experts.*.up_proj.weight", - ], - target_patterns="mlp.experts.gate_up_proj", - operations=[MergeModulelist(dim=0), Concatenate(dim=1)], - ), - WeightConverter( - source_patterns="mlp.experts.*.down_proj.weight", - target_patterns="mlp.experts.down_proj", - operations=[MergeModulelist(dim=0)], - ), - ] - - mapping["cohere_asr"] = [ - WeightRenaming(r"encoder\.pre_encode\.conv\.", r"encoder.subsampling.layers."), - WeightRenaming(r"encoder\.pre_encode\.out\.", r"encoder.subsampling.linear."), - WeightRenaming(r"transf_decoder\._embedding\.position_embedding\.pos_enc", r"decoder.pos_emb.weight"), - WeightRenaming(r"transf_decoder\._embedding\.token_embedding", r"decoder.embed_tokens"), - WeightRenaming(r"transf_decoder\._embedding\.layer_norm", r"decoder.embedding_layernorm"), - WeightRenaming(r"transf_decoder\._decoder\.final_layer_norm", r"decoder.norm"), - WeightRenaming(r"transf_decoder\._decoder\.layers", r"decoder.layers"), - WeightRenaming(r"encoder_decoder_proj\.", r"decoder.proj."), - WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_q", r"encoder.\1.self_attn.q_proj"), - WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_k", r"encoder.\1.self_attn.k_proj"), - WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_v", r"encoder.\1.self_attn.v_proj"), - WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_out", r"encoder.\1.self_attn.o_proj"), - WeightRenaming(r"encoder\.(.+)\.self_attn\.linear_pos", r"encoder.\1.self_attn.relative_k_proj"), - WeightRenaming(r"encoder\.(.+)\.self_attn\.pos_bias_u", r"encoder.\1.self_attn.bias_u"), - WeightRenaming(r"encoder\.(.+)\.self_attn\.pos_bias_v", r"encoder.\1.self_attn.bias_v"), - WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.query_net", r"decoder.\1.self_attn.q_proj"), - WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.key_net", r"decoder.\1.self_attn.k_proj"), - WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.value_net", r"decoder.\1.self_attn.v_proj"), - WeightRenaming(r"decoder\.(.+)\.first_sub_layer\.out_projection", r"decoder.\1.self_attn.o_proj"), - WeightRenaming(r"\.second_sub_layer\.query_net", r".encoder_attn.q_proj"), - WeightRenaming(r"\.second_sub_layer\.key_net", r".encoder_attn.k_proj"), - WeightRenaming(r"\.second_sub_layer\.value_net", r".encoder_attn.v_proj"), - WeightRenaming(r"\.second_sub_layer\.out_projection", r".encoder_attn.o_proj"), - WeightRenaming(r"\.third_sub_layer\.dense_in", r".mlp.fc1"), - WeightRenaming(r"\.third_sub_layer\.dense_out", r".mlp.fc2"), - WeightRenaming(r"\.layer_norm_1\.", r".input_layernorm."), - WeightRenaming(r"\.layer_norm_2\.", r".post_attention_layernorm."), - WeightRenaming(r"\.layer_norm_3\.", r".final_layernorm."), - WeightRenaming(r"\.conv\.batch_norm", r".conv.norm"), - WeightRenaming(r"log_softmax\.mlp\.layer0", r"proj_out"), - ] - for model_type, base_pattern in _MODEL_TO_CONVERSION_PATTERN.items(): if model_type in mapping: continue diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index b08aa558d795..399b0be222e9 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -56,6 +56,7 @@ "rjieba": "rjieba", "rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1", "ruff": "ruff==0.14.10", + "transformers-mlinter": "transformers-mlinter==0.1.0", "ty": "ty==0.0.20", "sacrebleu": "sacrebleu>=1.4.12,<2.0.0", "sacremoses": "sacremoses", diff --git a/src/transformers/generation/configuration_utils.py b/src/transformers/generation/configuration_utils.py index 308c42564295..9163333cade9 100644 --- a/src/transformers/generation/configuration_utils.py +++ b/src/transformers/generation/configuration_utils.py @@ -350,6 +350,11 @@ class GenerationConfig(PushToHubMixin): _original_object_hash: int | None def __init__(self, **kwargs): + # Snapshot of the attributes the caller explicitly provided (before the `kwargs.pop(...)` calls below + # consume them). Used by `validate()` to restrict "minor issue" warnings to flags actually set by the user, + # as opposed to defaults inherited from a model's `generation_config.json`. + user_set_attributes = set(kwargs.keys()) + # Parameters that control the length of the output self.max_length = kwargs.pop("max_length", None) self.max_new_tokens = kwargs.pop("max_new_tokens", None) @@ -466,7 +471,7 @@ def __init__(self, **kwargs): ) # Validate the values of the attributes - self.validate() + self.validate(user_set_attributes=user_set_attributes) def __hash__(self): return hash(self.to_json_string(ignore_metadata=True)) @@ -587,7 +592,7 @@ def _get_default_generation_params() -> dict[str, Any]: "diversity_penalty": 0.0, } - def validate(self, strict=False): + def validate(self, strict=False, user_set_attributes: set[str] | None = None): """ Validates the values of the attributes of the [`GenerationConfig`] instance. Raises exceptions in the presence of parameterization that can be detected as incorrect from the configuration instance alone. @@ -597,6 +602,11 @@ def validate(self, strict=False): Args: strict (bool): If True, raise an exception for any issues found. If False, only log issues. + user_set_attributes (set[str], *optional*): Names of attributes the caller explicitly provided. When + supplied, "minor issue" warnings about conflicting flag combinations (e.g. sampling-only flags set + while `do_sample=False`) only fire if the conflicting flag is in this set -- avoiding noisy warnings + when the value was inherited from a model's default `generation_config.json`. When `None`, all set + attributes are considered user-set (backward-compatible behavior for direct `validate()` calls). """ minor_issues = {} # format: {attribute_name: issue_description} @@ -636,47 +646,82 @@ def validate(self, strict=False): # Note that we check `is not True` in purpose. Boolean fields can also be `None` so we # have to be explicit. Value of `None` is same as having `False`, i.e. the default value + if self.do_sample is not True: greedy_wrong_parameter_msg = ( - "`do_sample` is set not to set `True`. However, `{flag_name}` is set to `{flag_value}` -- this flag is only " - "used in sample-based generation modes. You should set `do_sample=True` or unset `{flag_name}`." + "`do_sample` is set to `{do_sample}`. However, `{flag_name}` is set to `{flag_value}` -- this flag is " + "only used in sample-based generation modes. You should set `do_sample=True` or unset `{flag_name}`." ) - if self.temperature is not None and self.temperature != 1.0: + + # The warnings are suppressed for flags that weren't explicitly set by the caller when `do_sample=False` is explicitly + # required by the user: values such as `top_p` inherited from a model's `generation_config.json` are harmless when + # the user opts for greedy decoding + def _should_warn(attr: str) -> bool: + do_sample_set = user_set_attributes is not None and "do_sample" in user_set_attributes + attr_set = user_set_attributes is not None and attr in user_set_attributes + # We should warn only if both are explicitly set, none are set, or only the new attr is set while `do_sample` is already False + return ( + (do_sample_set and attr_set) + or (not do_sample_set and not attr_set) + or (attr_set and not do_sample_set) + ) + + if self.temperature is not None and self.temperature != 1.0 and _should_warn("temperature"): minor_issues["temperature"] = greedy_wrong_parameter_msg.format( - flag_name="temperature", flag_value=self.temperature + do_sample=self.do_sample, flag_name="temperature", flag_value=self.temperature + ) + if self.top_p is not None and self.top_p != 1.0 and _should_warn("top_p"): + minor_issues["top_p"] = greedy_wrong_parameter_msg.format( + do_sample=self.do_sample, flag_name="top_p", flag_value=self.top_p ) - if self.top_p is not None and self.top_p != 1.0: - minor_issues["top_p"] = greedy_wrong_parameter_msg.format(flag_name="top_p", flag_value=self.top_p) - if self.min_p is not None: - minor_issues["min_p"] = greedy_wrong_parameter_msg.format(flag_name="min_p", flag_value=self.min_p) - if self.top_h is not None: - minor_issues["top_h"] = greedy_wrong_parameter_msg.format(flag_name="top_h", flag_value=self.top_h) - if self.typical_p is not None and self.typical_p != 1.0: + if self.min_p is not None and _should_warn("min_p"): + minor_issues["min_p"] = greedy_wrong_parameter_msg.format( + do_sample=self.do_sample, flag_name="min_p", flag_value=self.min_p + ) + if self.top_h is not None and _should_warn("top_h"): + minor_issues["top_h"] = greedy_wrong_parameter_msg.format( + do_sample=self.do_sample, flag_name="top_h", flag_value=self.top_h + ) + if self.typical_p is not None and self.typical_p != 1.0 and _should_warn("typical_p"): minor_issues["typical_p"] = greedy_wrong_parameter_msg.format( - flag_name="typical_p", flag_value=self.typical_p + do_sample=self.do_sample, flag_name="typical_p", flag_value=self.typical_p + ) + if self.top_k is not None and self.top_k != 50 and _should_warn("top_k"): + minor_issues["top_k"] = greedy_wrong_parameter_msg.format( + do_sample=self.do_sample, flag_name="top_k", flag_value=self.top_k ) - if self.top_k is not None and self.top_k != 50: - minor_issues["top_k"] = greedy_wrong_parameter_msg.format(flag_name="top_k", flag_value=self.top_k) - if self.epsilon_cutoff is not None and self.epsilon_cutoff != 0.0: + if self.epsilon_cutoff is not None and self.epsilon_cutoff != 0.0 and _should_warn("epsilon_cutoff"): minor_issues["epsilon_cutoff"] = greedy_wrong_parameter_msg.format( - flag_name="epsilon_cutoff", flag_value=self.epsilon_cutoff + do_sample=self.do_sample, flag_name="epsilon_cutoff", flag_value=self.epsilon_cutoff ) - if self.eta_cutoff is not None and self.eta_cutoff != 0.0: + if self.eta_cutoff is not None and self.eta_cutoff != 0.0 and _should_warn("eta_cutoff"): minor_issues["eta_cutoff"] = greedy_wrong_parameter_msg.format( - flag_name="eta_cutoff", flag_value=self.eta_cutoff + do_sample=self.do_sample, flag_name="eta_cutoff", flag_value=self.eta_cutoff ) - # 2.2. detect beam-only parameterization when not in beam mode + # 2.2. detect beam-only parameterization when not in beam mode. Same provenance filtering as above -- + # both `num_beams` and the beam-only flag must be user-set for the warning to fire. if self.num_beams is None or self.num_beams == 1: single_beam_wrong_parameter_msg = ( - "`num_beams` is set to {num_beams}. However, `{flag_name}` is set to `{flag_value}` -- this flag is only used " - "in beam-based generation modes. You should set `num_beams>1` or unset `{flag_name}`." + "`num_beams` is set to {num_beams}. However, `{flag_name}` is set to `{flag_value}` -- this flag is " + "only used in beam-based generation modes. You should set `num_beams>1` or unset `{flag_name}`." ) - if self.early_stopping is not None and self.early_stopping is not False: + + def _should_warn(attr: str) -> bool: + num_beams_set = user_set_attributes is not None and "num_beams" in user_set_attributes + attr_set = user_set_attributes is not None and attr in user_set_attributes + # We should warn only if both are explicitly set, none are set, or only the new attr is set while `num_beams` is already 1 + return ( + (num_beams_set and attr_set) + or (not num_beams_set and not attr_set) + or (attr_set and not num_beams_set) + ) + + if self.early_stopping is not None and self.early_stopping is not False and _should_warn("early_stopping"): minor_issues["early_stopping"] = single_beam_wrong_parameter_msg.format( num_beams=self.num_beams, flag_name="early_stopping", flag_value=self.early_stopping ) - if self.length_penalty is not None and self.length_penalty != 1.0: + if self.length_penalty is not None and self.length_penalty != 1.0 and _should_warn("length_penalty"): minor_issues["length_penalty"] = single_beam_wrong_parameter_msg.format( num_beams=self.num_beams, flag_name="length_penalty", flag_value=self.length_penalty ) @@ -1232,8 +1277,9 @@ def update(self, defaults_only=False, allow_custom_entries=False, **kwargs): setattr(self, key, value) to_remove.append(key) - # Confirm that the updated instance is still valid - self.validate() + # Confirm that the updated instance is still valid. Only attributes *explicitly* updated in this call count + # as user-set for warning purposes: defaults inherited from a model's config shouldn't emit warnings. + self.validate(user_set_attributes=set(to_remove)) # Remove all the attributes that were updated, without modifying the input dict unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove} diff --git a/src/transformers/integrations/fouroversix.py b/src/transformers/integrations/fouroversix.py index db741af601ff..a79e213b7055 100644 --- a/src/transformers/integrations/fouroversix.py +++ b/src/transformers/integrations/fouroversix.py @@ -59,14 +59,17 @@ def convert( def adapt_fouroversix_config(config: FourOverSixConfig): return ModelQuantizationConfig( + activation_dtype=config.activation_dtype, activation_scale_rule=config.activation_scale_rule, dtype=config.dtype, + gradient_dtype=config.gradient_dtype, gradient_scale_rule=config.gradient_scale_rule, keep_master_weights=config.keep_master_weights, matmul_backend=config.matmul_backend, output_dtype=config.output_dtype, quantize_backend=config.quantize_backend, scale_rule=config.scale_rule, + weight_dtype=config.weight_dtype, weight_scale_2d=config.weight_scale_2d, weight_scale_rule=config.weight_scale_rule, modules_to_not_convert=config.modules_to_not_convert, diff --git a/src/transformers/integrations/ggml.py b/src/transformers/integrations/ggml.py index 29ec365e7ce2..c9ba021c54db 100644 --- a/src/transformers/integrations/ggml.py +++ b/src/transformers/integrations/ggml.py @@ -89,6 +89,21 @@ "expert_count": "num_experts", "expert_used_count": "num_experts_per_tok", }, + "gpt_oss": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + "expert_count": "num_local_experts", + "expert_used_count": "num_experts_per_tok", + "sliding_window": "sliding_window", + }, "lfm2": { "context_length": "max_position_embeddings", "block_count": "num_hidden_layers", diff --git a/src/transformers/integrations/hub_kernels.py b/src/transformers/integrations/hub_kernels.py index 88aff578fdc6..b1e6c74ddf10 100644 --- a/src/transformers/integrations/hub_kernels.py +++ b/src/transformers/integrations/hub_kernels.py @@ -225,9 +225,12 @@ def use_kernel_func_from_hub(func_name: str): ) }, "cuda": { + Mode.TRAINING: FuncRepository( + repo_id="kernels-community/rotary", func_name="apply_rotary_transformers" + ), Mode.INFERENCE: FuncRepository( repo_id="kernels-community/rotary", func_name="apply_rotary_transformers" - ) + ), }, } @@ -438,11 +441,13 @@ def get_kernel( def use_kernelized_func(module_names: list[Callable] | Callable): """ - This decorator attaches the target function as an attribute of the module. - The function must already be decorated with @use_kernel_func_from_hub - this decorator then wraps it as an nn.Module internally. - When kernelize is later applied to the full model, the function can be accessed as a regular module attribute and kernelized just like any other layer. - The kernelization is performed in place, modifying the module directly. + This decorator attaches the target function within the module as a plain attribute (not as a submodule). + Keep in mind that this registration is only meant for `kernelize` to recognize its target modules (i.e. + function exchanged for a weightless `nn.Module` with the same forward) to then exchange to the kernel + variation (in-place) if the conditions are met. + + We cache each of these function-based registrations: After proper registration and exchange it is removed + from the module's `_modules` dict as it does not really act as `nn.Module` but a base function. """ if isinstance(module_names, Callable): module_names = [module_names] @@ -452,18 +457,20 @@ def decorator(cls): def new_init(self, *args, **kwargs): orig_init(self, *args, **kwargs) - # Skip attaching the kernelized submodule under DeepSpeed ZeRO-3: the coordinator traces - # the module graph at init time, and a child `nn.Module` that is not actually invoked - # during forward (e.g. when the model keeps calling the plain Python `apply_rotary_pos_emb`) - # breaks the parameter fetch trace and raises `IndexError: pop from an empty deque`. - # See https://github.com/huggingface/transformers/issues/45137 - from .deepspeed import is_deepspeed_zero3_enabled - - if is_deepspeed_zero3_enabled(): - return + + # Register new function as non-submodule within the modules dict + hidden_kernels = self.__dict__.setdefault("_hidden_kernels", {}) for fn in module_names: - # we hardcode the name of the function to "rotary_fn" for now - setattr(self, "rotary_fn", fn) + name = ( + getattr(fn, "__name__", None) + or getattr(fn, "kernel_layer_name", None) + or getattr(fn, "func_name", None) + ) + if name is None: + raise ValueError(f"Could not infer kernel function name for {fn!r}") + + # Do not register as submodule! Hide it behind a dict to be removed later after registering it + hidden_kernels[name] = fn cls.__init__ = new_init return cls diff --git a/src/transformers/integrations/moe.py b/src/transformers/integrations/moe.py index 70178dd1fa7e..d17522d26daa 100644 --- a/src/transformers/integrations/moe.py +++ b/src/transformers/integrations/moe.py @@ -383,6 +383,10 @@ def grouped_mm_experts_forward( sample_weights = top_k_weights.reshape(-1) # (S,) expert_ids = top_k_index.reshape(-1) # (S,) + # Handle invalid expert IDs from Expert Parallelism (EP) + invalid_mask = expert_ids >= self.num_experts + expert_ids = expert_ids.clamp(0, self.num_experts - 1) + # Sort by expert for grouped processing perm = torch.argsort(expert_ids) inv_perm = torch.empty_like(perm) @@ -433,8 +437,10 @@ def grouped_mm_experts_forward( proj_out, selected_weights, offsets, bias=selected_biases, is_transposed=self.is_transposed ) # (S, hidden_dim) - # Apply routing weights + # Apply routing weights and zero out invalid expert contributions from EP weighted_out = proj_out * sample_weights_g.unsqueeze(-1) # (S, hidden_dim) + invalid_mask_g = invalid_mask[perm] + weighted_out.masked_fill_(invalid_mask_g.unsqueeze(-1), 0.0) # Restore original order weighted_out = weighted_out[inv_perm] # (S, hidden_dim) diff --git a/src/transformers/integrations/tensor_parallel.py b/src/transformers/integrations/tensor_parallel.py index 39a2e696941b..82d6d284f052 100644 --- a/src/transformers/integrations/tensor_parallel.py +++ b/src/transformers/integrations/tensor_parallel.py @@ -1052,10 +1052,8 @@ def shard_tensor( ) local_num_experts = global_num_experts // self.device_mesh.size() shard_size = local_num_experts - if isinstance(device, torch.device): - device = device.index if device.index is not None else 0 - start = device * shard_size - end = (device + 1) * shard_size + start = self.rank * shard_size + end = (self.rank + 1) * shard_size # special case we don't "shard" just send this entire tensor to the correct rank. shape = param.get_shape() if not isinstance(param, torch.Tensor) else param.shape if tensor_idx is not None and start <= tensor_idx < end: @@ -1078,7 +1076,7 @@ def get_expected_sharded_shape(self, full_shape: tuple[int, ...] | torch.Size) - def update_module_attributes(self, module: nn.Module): if hasattr(module, "num_experts"): - module.num_experts = self.get_expected_sharded_shape((module.num_experts,))[0] + module.num_experts = self.get_expected_sharded_shape((self.empty_param.shape[0],))[0] class RouterParallel(TensorParallelLayer): @@ -1094,49 +1092,66 @@ def _prepare_input_fn(self, mod, inputs, device_mesh): def _prepare_output_fn(self, mod, outputs, device_mesh): """ - Imagine if you had 4 tokens, top_k = 4, and 128experts. - With EP = 8. The num_local_expert should be 128/8 = 16 - Imagine router_indices being: - [ 52, 42, 119, 67], - [102, 89, 61, 40], - [ 82, 103, 4, 34], - [ 93, 23, 109, 11], - - then you can map which rank should be getting which values - - [3, 2, 7, 4], - [6, 5, 3, 2], - [5, 6, 0, 2], - [5, 1, 6, 0], - - Thus for say rank 0, you fill with 16 (num_local_expert) the index tensor - - [ 16, 16, 16, 16], - [ 16, 16, 16, 16], - [ 16, 16, 4, 16], - [ 16, 16, 16, 11], - - This works well. For another rank you need to make sure you round to num_local_expert - because the next operation will one hot encode the router index vector. - - This allows us to know directly which local expert is hit. - Similarly the scores are indexed with something created form - router_indices. - - The kinda naive training loop that we use for device_map "auto" uses a similar logic. - Here we are just making each rank believe that he is alone, and he computes his part of the hiddenstates. - Mask invalid indices with num_local_expert for one-hot encoding, so the computes will skip the masking index. + Remap global expert indices to local and zero out non-local scores. + + Example: 4 tokens, top_k=4, 128 experts, EP=8. num_local_experts = 128/8 = 16. + + Router produces (all ranks see the same values): + router_scores: (4, 4) — top-k routing weights + router_indices: (4, 4) — global expert IDs + [ 52, 42, 119, 67], + [102, 89, 61, 40], + [ 82, 103, 4, 34], + [ 93, 23, 109, 11], + + Each index maps to a rank: index // 16 gives the owning rank. + [3, 2, 7, 4], + [6, 5, 3, 2], + [5, 6, 0, 2], + [5, 1, 6, 0], + + For rank 0 (owns experts 0-15), we remap local indices with fmod and + fill non-local with sentinel=16 (used for one_hot masking): + router_indices (rank 0): + [ 16, 16, 16, 16], + [ 16, 16, 16, 16], + [ 16, 16, 4, 16], + [ 16, 16, 16, 11], + + Scores for non-local experts are zeroed out via masked_fill: + router_scores (rank 0): + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.3, 0.0], ← only expert 4 (local) keeps its score + [0.0, 0.0, 0.0, 0.1], ← only expert 11 (local) keeps its score + + both router_scores and router_indices stay (seq, top_k) shape. + They are paired element-wise: scores[i] is the weight for indices[i]. + All expert forward implementations (grouped_mm, batched_mm, eager) flatten + both with reshape(-1) and rely on this pairing. Changing the shape of one + without the other breaks routing! + + Each rank believes it is alone and computes only its part of the hidden states. + The sentinel index (num_local_experts) is skipped by one_hot encoding or clamped + + masked in grouped_mm/batched_mm. After the expert forward, an all_reduce sums + partial outputs across EP ranks to produce the full result. """ ep_rank, ep_size = device_mesh.get_local_rank(), device_mesh.size() - if mod.num_experts % ep_size != 0: + num_experts = getattr(mod, "num_experts", None) + if num_experts is None: + num_experts = getattr(getattr(mod, "config", None), "num_experts", None) + if num_experts is None: + raise AttributeError(f"Router module {type(mod).__name__} is missing num_experts and config.num_experts") + + if num_experts % ep_size != 0: raise ValueError( - f"The number of experts must be divisible by number of ep_size: {mod.num_experts} % {ep_size} != 0" + f"The number of experts must be divisible by number of ep_size: {num_experts} % {ep_size} != 0" ) - num_local_experts = mod.num_experts // ep_size + num_local_experts = num_experts // ep_size router_logits, router_scores, router_indices = outputs - router_scores = torch.zeros_like(router_logits).scatter_(1, router_indices, router_scores) - router_scores = router_scores[:, ep_rank * num_local_experts : (ep_rank + 1) * num_local_experts] - router_indices = router_indices.masked_fill((router_indices // num_local_experts) != ep_rank, -1) + non_local_mask = (router_indices // num_local_experts) != ep_rank + router_scores = router_scores.masked_fill(non_local_mask, 0.0) + router_indices = router_indices.masked_fill(non_local_mask, -1) # As -1 % 1 is 0, we can only use mask fill when num_local_experts is 1 if num_local_experts > 1: router_indices = torch.fmod(router_indices, num_local_experts) diff --git a/src/transformers/modeling_gguf_pytorch_utils.py b/src/transformers/modeling_gguf_pytorch_utils.py index 66306b6f71f6..2de6cc13fc85 100644 --- a/src/transformers/modeling_gguf_pytorch_utils.py +++ b/src/transformers/modeling_gguf_pytorch_utils.py @@ -171,6 +171,107 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st out.copy_(torch_weights) +class GptOssTensorProcessor(TensorProcessor): + """ + Tensor processor for GPT-OSS models (MoE with 128 experts). + Handles: + - Splitting stacked expert tensors (down_proj, gate_proj, up_proj) into individual experts. + - Interleaving gate and up projections if stored in a combined tensor (gate_up_projs). + - Bias tensors (1D) are passed through without transpose. + """ + + # Regex for separate expert tensors: e.g., blk.0.ffn_down_projs.weight + GGUF_MOE_WEIGHTS_PATTERN = re.compile(r"blk\.(?P\d+)\.ffn_(?Pdown|gate|up)_projs\.weight$") + # Regex for combined gate+up tensor: e.g., blk.0.ffn_gate_up_projs.weight + GGUF_MOE_COMBINED_PATTERN = re.compile(r"blk\.(?P\d+)\.ffn_gate_up_projs\.weight$") + + def __init__(self, config=None): + super().__init__(config=config) + + def process(self, weights, name: str, **kwargs): + # 1. Handle separate MoE expert tensors (down, gate, up) + if m := self.GGUF_MOE_WEIGHTS_PATTERN.match(name): + tensor_key_mapping = kwargs.get("tensor_key_mapping") + parsed_parameters = kwargs.get("parsed_parameters") + if tensor_key_mapping and parsed_parameters: + self._split_moe_expert_tensor(weights, parsed_parameters, m["bid"], m["proj"], tensor_key_mapping) + return GGUFTensor(weights, None, {}) # signal handled + + # 2. Handle combined gate+up tensor + if m := self.GGUF_MOE_COMBINED_PATTERN.match(name): + tensor_key_mapping = kwargs.get("tensor_key_mapping") + parsed_parameters = kwargs.get("parsed_parameters") + if tensor_key_mapping and parsed_parameters: + self._interleave_gate_up_tensor(weights, parsed_parameters, m["bid"], tensor_key_mapping) + return GGUFTensor(weights, None, {}) + + # 3. Bias tensors (1D) → no transpose + if ".bias" in name and len(weights.shape) == 1: + return GGUFTensor(weights, name, {}) + + # 4. Default handling for all other tensors + return GGUFTensor(weights, name, {}) + + def _split_moe_expert_tensor( + self, + weights: np.ndarray, + parsed_parameters: dict, + bid: str, + proj: str, + tensor_key_mapping: dict, + ): + """Split a stacked MoE tensor into individual expert tensors.""" + num_experts = self.config.get("num_local_experts", 128) + # Expected shape: [num_experts, hidden_size, intermediate_size] (or swapped). + # We assume the stored order is correct for the projection after splitting. + for i in range(min(num_experts, weights.shape[0])): + expert_weight = weights[i] # shape: [hidden, inter] or [inter, hidden] + # Build HF parameter name + hf_name = f"model.layers.{bid}.block_sparse_moe.experts.{i}.{proj}_proj.weight" + # Apply any user‑provided tensor key mapping + for key, mapped_key in tensor_key_mapping.items(): + if key in hf_name: + hf_name = hf_name.replace(key, mapped_key) + # Store the tensor + parsed_parameters["tensors"][hf_name] = torch.tensor(expert_weight, copy=True) + + def _interleave_gate_up_tensor( + self, + weights: np.ndarray, + parsed_parameters: dict, + bid: str, + tensor_key_mapping: dict, + ): + """ + Process a combined gate+up tensor. + Expected shape: [num_experts, intermediate_size, hidden_size]. + Interleaving: gate occupies first half of intermediate dimension, + up occupies second half. Transpose to [hidden, half_inter] per expert. + """ + num_experts = self.config.get("num_local_experts", 128) + inter_size = weights.shape[1] + half_inter = inter_size // 2 + gate_part = weights[:, :half_inter, :] # [E, half_inter, hidden] + up_part = weights[:, half_inter:, :] # [E, half_inter, hidden] + + for i in range(min(num_experts, weights.shape[0])): + gate_weight = gate_part[i].T # [hidden, half_inter] + up_weight = up_part[i].T # [hidden, half_inter] + + gate_name = f"model.layers.{bid}.block_sparse_moe.experts.{i}.gate_proj.weight" + up_name = f"model.layers.{bid}.block_sparse_moe.experts.{i}.up_proj.weight" + + # Apply mapping + for key, mapped_key in tensor_key_mapping.items(): + if key in gate_name: + gate_name = gate_name.replace(key, mapped_key) + if key in up_name: + up_name = up_name.replace(key, mapped_key) + + parsed_parameters["tensors"][gate_name] = torch.tensor(gate_weight, copy=True) + parsed_parameters["tensors"][up_name] = torch.tensor(up_weight, copy=True) + + class BloomTensorProcessor(TensorProcessor): def __init__(self, config=None): super().__init__(config=config) @@ -355,6 +456,7 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st TENSOR_PROCESSORS = { "llama": LlamaTensorProcessor, "qwen2moe": Qwen2MoeTensorProcessor, + "gpt_oss": GptOssTensorProcessor, "qwen3moe": Qwen2MoeTensorProcessor, "bloom": BloomTensorProcessor, "t5": T5TensorProcessor, @@ -416,6 +518,8 @@ def get_gguf_hf_weights_map( model_type = "t5" elif model_type == "minimax_m2": model_type = "minimax-m2" + elif model_type == "gpt_oss": + model_type = "gpt-oss" arch = None for key, value in MODEL_ARCH_NAMES.items(): if value == model_type: @@ -463,7 +567,7 @@ def get_gguf_hf_weights_map( return gguf_to_hf_name_map -def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_load=None): +def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_load=None, torch_dtype=None): """ Load a GGUF file and return a dictionary of parsed parameters containing tensors, the parsed tokenizer and config attributes. @@ -474,6 +578,12 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo return_tensors (`bool`, defaults to `False`): Whether to read the tensors from the file and return them. Not doing so is faster and only loads the metadata in memory. + model_to_load (`nn.Module`, *optional*): + The model to load the weights into. This is used to map GGUF tensor names to + Transformers parameter names. + torch_dtype (`torch.dtype`, *optional*): + The desired `torch.dtype` for the loaded tensors. If provided, tensors will be + converted to this dtype immediately after dequantization to save memory. """ if is_gguf_available() and is_torch_available(): from gguf import GGUFReader, dequantize @@ -516,6 +626,8 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo if "qwen2moe" in architecture: updated_architecture = "qwen2_moe" + elif "gpt_oss" in architecture or "gpt-oss" in architecture: + updated_architecture = "gpt_oss" elif "qwen3moe" in architecture: updated_architecture = "qwen3_moe" elif "minimax-m2" in architecture: @@ -603,6 +715,45 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo i for i, num_kv_heads in enumerate(gguf_num_key_value_heads) if num_kv_heads > 0 ] + if updated_architecture == "gpt_oss": + # Helper to read keys with the correct prefix + def read_gpt_key(reader, suffix, default=None): + key = f"gpt-oss.{suffix}" + if key in reader.fields: + val = reader.fields[key].parts[0] + if isinstance(val, bytes): + val = val.decode("utf-8") + return val + return default + + # Reconstruct rope_scaling from GGUF metadata + rope_type = read_gpt_key(reader, "rope.scaling.type") + if rope_type is not None: + rope_scaling = {"rope_type": rope_type} + + # Collect all rope.scaling keys dynamically + for key in reader.fields: + if not key.startswith("gpt-oss.rope.scaling."): + continue + suffix = key[len("gpt-oss.rope.scaling.") :] + if suffix == "type": + continue + value = reader.fields[key].parts[0] + if isinstance(value, bytes): + value = value.decode("utf-8") + # Convert to appropriate type + if suffix in ("factor", "attention_factor", "beta_fast", "beta_slow"): + value = float(value) + elif suffix in ("original_context_length", "original_max_position_embeddings"): + # Map GGUF's original_context_length to HF's original_max_position_embeddings + suffix = "original_max_position_embeddings" + value = int(value) + else: + pass + rope_scaling[suffix] = value + + parsed_parameters["config"]["rope_scaling"] = rope_scaling + # retrieve config vocab_size from tokenizer # Please refer to https://github.com/huggingface/transformers/issues/32526 for more details if "vocab_size" not in parsed_parameters["config"]: @@ -644,7 +795,10 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo name = tensor_key_mapping[name] - parsed_parameters["tensors"][name] = torch.from_numpy(np.copy(weights)) + tensor = torch.from_numpy(np.copy(weights)) + if torch_dtype is not None: + tensor = tensor.to(torch_dtype) + parsed_parameters["tensors"][name] = tensor if len(reader_keys) > 0: logger.info(f"Some keys of the GGUF file were not considered: {reader_keys}") diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 2e98863a762d..eb092019b678 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -36,6 +36,7 @@ from huggingface_hub import create_repo, is_offline_mode, split_torch_state_dict_into_shards from packaging import version from safetensors import safe_open +from safetensors.torch import load as _safe_load_bytes from safetensors.torch import save_file as safe_save_file from torch import Tensor, nn from torch.distributions import constraints @@ -178,6 +179,7 @@ class LoadStateDictConfig: device_mesh: Optional["torch.distributed.device_mesh.DeviceMesh"] = None weights_only: bool = True weight_mapping: list[WeightConverter | WeightRenaming] | None = None + disable_mmap: bool | None = None @property def is_quantized(self) -> bool: @@ -288,14 +290,54 @@ def get_state_dict_dtype(state_dict): } +def _is_on_hf_mount(path: "str | os.PathLike") -> bool: + """True if `path` lives on an hf-mount FUSE filesystem (device string 'hf-mount'). + + hf-mount's mmap + readahead interaction deadlocks under parallel page-faults, + so callers should load the file into memory instead. Linux-only; returns False + on other platforms. + """ + if not sys.platform.startswith("linux"): + return False + try: + real = os.path.realpath(os.fspath(path)) + with open("/proc/mounts", encoding="utf-8") as fh: + entries = sorted( + ((p[0], p[1]) for p in (l.split() for l in fh) if len(p) >= 2), + key=lambda e: len(e[1]), + reverse=True, + ) + for dev, mp in entries: + if real == mp or real.startswith(mp.rstrip("/") + "/"): + return dev == "hf-mount" + except (OSError, ValueError): + pass + return False + + def load_state_dict( - checkpoint_file: str | os.PathLike, map_location: str | torch.device = "cpu", weights_only: bool = True + checkpoint_file: str | os.PathLike, + map_location: str | torch.device = "cpu", + weights_only: bool = True, + disable_mmap: bool | None = None, ) -> dict[str, torch.Tensor]: """ Reads a `safetensor` or a `.bin` checkpoint file. We load the checkpoint on "cpu" by default. + + When `disable_mmap` is True, safetensors files are read fully into memory instead of + being memory-mapped. When `disable_mmap` is None (default), it is auto-detected to True + on hf-mount FUSE filesystems (see `_is_on_hf_mount`). """ + if disable_mmap is None: + disable_mmap = _is_on_hf_mount(checkpoint_file) # Use safetensors if possible if checkpoint_file.endswith(".safetensors"): + if disable_mmap and map_location != "meta": + with open(checkpoint_file, "rb") as _fh: + state_dict = _safe_load_bytes(_fh.read()) + if map_location != "cpu": + state_dict = {k: v.to(map_location) for k, v in state_dict.items()} + return state_dict with safe_open(checkpoint_file, framework="pt") as f: state_dict = {} for k in f.keys(): @@ -782,14 +824,16 @@ def _get_dtype( dtype = sharded_metadata["dtype"] elif state_dict is not None: dtype = get_state_dict_dtype(state_dict) + elif checkpoint_files is not None and checkpoint_files[0].endswith(".gguf"): + dtype = torch.float32 else: state_dict = load_state_dict( checkpoint_files[0], map_location="meta", weights_only=weights_only ) dtype = get_state_dict_dtype(state_dict) logger.info( - "Since the `dtype` attribute can't be found in model's config object, " - "will use dtype={dtype} as derived from model's weights" + f"Since the `dtype` attribute can't be found in model's config object, " + f"will use dtype={dtype} as derived from model's weights" ) elif hasattr(torch, dtype): dtype = getattr(torch, dtype) @@ -1949,9 +1993,9 @@ def _can_set_attn_implementation(cls) -> bool: """Detect whether the class supports setting its attention implementation dynamically. It is an ugly check based on opening the file, but avoids maintaining yet another property flag. """ - class_module = sys.modules[cls.__module__] - # This can happen for a custom model in a jupyter notebook or repl for example - simply do not allow to set it then - if not hasattr(class_module, "__file__"): + class_module = sys.modules.get(cls.__module__) + # Missing module entry (e.g. cleared by a test) or custom model in a jupyter notebook / repl -> do not allow to set it + if class_module is None or not hasattr(class_module, "__file__"): return False class_file = class_module.__file__ with open(class_file, "r", encoding="utf-8") as f: @@ -1968,9 +2012,9 @@ def _can_set_experts_implementation(cls) -> bool: """Detect whether the class supports setting its experts implementation dynamically. It is an ugly check based on opening the file, but avoids maintaining yet another property flag. """ - class_module = sys.modules[cls.__module__] - # This can happen for a custom model in a jupyter notebook or repl for example - simply do not allow to set it then - if not hasattr(class_module, "__file__"): + class_module = sys.modules.get(cls.__module__) + # Missing module entry (e.g. cleared by a test) or custom model in a jupyter notebook / repl -> do not allow to set it + if class_module is None or not hasattr(class_module, "__file__"): return False class_file = class_module.__file__ with open(class_file, "r", encoding="utf-8") as f: @@ -3697,6 +3741,7 @@ def from_pretrained( use_safetensors: bool | None = None, weights_only: bool = True, fusion_config: dict[str, bool | dict[str, Any]] | None = None, + disable_mmap: bool | None = None, **kwargs, ) -> SpecificPreTrainedModelType: r""" @@ -3875,6 +3920,12 @@ def from_pretrained( Indicates whether unpickler should be restricted to loading only tensors, primitive types, dictionaries and any types added via torch.serialization.add_safe_globals(). When set to False, we can load wrapper tensor subclass weights. + disable_mmap (`bool`, *optional*): + Whether to disable memory mapping when loading safetensors checkpoints. When `None` (default), + it is auto-detected to `True` when the checkpoint lives on an `hf-mount` FUSE filesystem + (used by HF Spaces/Endpoints), where mmap + parallel page-faults can deadlock. When `True`, + files are read fully into memory and parsed with `safetensors.torch.load`. When `False`, the + default memory-mapped loader is always used. fusion_config (`dict[str, bool | dict[str, Any]]`, *optional*): Optional fusion configuration applied before model instantiation. Each key enables a fusion family and its value can either be `True` to enable that fusion with default options or a dictionary of @@ -4078,6 +4129,11 @@ def from_pretrained( is_quantized = hf_quantizer is not None + # Find the correct dtype based on current state + config, dtype = _get_dtype( + dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only, hf_quantizer + ) + if gguf_file: from .modeling_gguf_pytorch_utils import load_gguf_checkpoint @@ -4085,14 +4141,10 @@ def from_pretrained( # passed directly as a kwarg from now on with torch.device("meta"): dummy_model = cls(config) - state_dict = load_gguf_checkpoint(checkpoint_files[0], return_tensors=True, model_to_load=dummy_model)[ - "tensors" - ] - # Find the correct dtype based on current state - config, dtype = _get_dtype( - dtype, checkpoint_files, config, sharded_metadata, state_dict, weights_only, hf_quantizer - ) + state_dict = load_gguf_checkpoint( + checkpoint_files[0], return_tensors=True, model_to_load=dummy_model, torch_dtype=dtype + )["tensors"] config.name_or_path = pretrained_model_name_or_path @@ -4153,6 +4205,7 @@ def from_pretrained( weight_mapping=weight_conversions, use_safetensors=use_safetensors, download_kwargs=download_kwargs, + disable_mmap=disable_mmap, ) loading_info, disk_offload_index = cls._load_pretrained_model(model, state_dict, checkpoint_files, load_config) loading_info = cls._finalize_model_loading(model, load_config, loading_info) @@ -4243,7 +4296,12 @@ def _load_pretrained_model( merged_state_dict = {} for ckpt_file in checkpoint_files: merged_state_dict.update( - load_state_dict(ckpt_file, map_location="cpu", weights_only=load_config.weights_only) + load_state_dict( + ckpt_file, + map_location="cpu", + weights_only=load_config.weights_only, + disable_mmap=load_config.disable_mmap, + ) ) state_dict = merged_state_dict error_msgs, missing_keys = _load_state_dict_into_zero3_model(model, state_dict, load_config) @@ -4262,6 +4320,10 @@ def _load_pretrained_model( elif checkpoint_files is not None and checkpoint_files[0].endswith(".safetensors") and state_dict is None: merged_state_dict = {} for file in checkpoint_files: + if load_config.disable_mmap or _is_on_hf_mount(file): + with open(file, "rb") as _fh: + merged_state_dict.update(_safe_load_bytes(_fh.read())) + continue file_pointer = safe_open(file, framework="pt", device="cpu") all_pointer.add(file_pointer) for k in file_pointer.keys(): @@ -4270,7 +4332,7 @@ def _load_pretrained_model( elif checkpoint_files is not None: merged_state_dict = {} for ckpt_file in checkpoint_files: - merged_state_dict.update(load_state_dict(ckpt_file)) + merged_state_dict.update(load_state_dict(ckpt_file, disable_mmap=load_config.disable_mmap)) else: raise ValueError("Neither a state dict nor checkpoint files were found.") @@ -4278,7 +4340,7 @@ def _load_pretrained_model( model=model, state_dict=merged_state_dict, load_config=load_config, - tp_plan=model._tp_plan, + tp_plan=model.tp_plan, disk_offload_index=disk_offload_index, ) @@ -4466,15 +4528,31 @@ def loss_function(self, value): self._loss_function = value def kernelize(self, mode=None): + """Temporarily register hidden kernel wrappers so `kernelize` can discover and replace them.""" if not is_kernels_available(): raise ValueError( - "Kernels are not available. To use kernels, please install kernels using `pip install kernels`" + "Kernels are not available. To use kernels, please install kernels using `pip install -U kernels`" ) from kernels import Device, Mode, kernelize - mode = Mode.INFERENCE if not self.training else Mode.TRAINING if mode is None else mode - kernelize(self, device=Device(type=self.device.type), mode=mode) - self._use_kernels = True + def attach_hidden_kernels(module): + for name, fn in getattr(module, "_hidden_kernels", {}).items(): + if name not in dict(module.named_children()): + module.register_module(name, fn) + + def detach_hidden_kernels(module): + for name in getattr(module, "_hidden_kernels", {}): + delattr(module, name) + + try: + self.apply(attach_hidden_kernels) + + mode = Mode.INFERENCE if not self.training else Mode.TRAINING if mode is None else mode + kernelize(self, device=Device(type=self.device.type), mode=mode) + self._use_kernels = True + + finally: + self.apply(detach_hidden_kernels) @property def use_kernels(self) -> bool: diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 1fbbc733c308..888b3b1c29c3 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -408,6 +408,7 @@ def forward(self, audio_features): ) class AudioFlamingo3ForConditionalGeneration(AudioFlamingo3PreTrainedModel, GenerationMixin): _keep_in_fp32_modules_strict = None + _supports_attention_backend = True _tp_plan = None _pp_plan = None diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index c325bc85300e..bbe4090b06ea 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -142,6 +142,7 @@ def __init__(self, config: AudioFlamingo3Config): """ ) class AudioFlamingo3ForConditionalGeneration(VoxtralForConditionalGeneration): + _supports_attention_backend = True _tp_plan = None _pp_plan = None _keep_in_fp32_modules_strict = None diff --git a/src/transformers/models/auto/auto_factory.py b/src/transformers/models/auto/auto_factory.py index 2f93efb1f57e..5ef09f8eb443 100644 --- a/src/transformers/models/auto/auto_factory.py +++ b/src/transformers/models/auto/auto_factory.py @@ -383,7 +383,14 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike[str], elif has_local_code: model_class = _get_model_class(config, cls._model_mapping) if model_class.config_class == config.sub_configs.get("text_config", None): + # TODO: Validate that copying the parent quantization config to the text sub-config preserves + # modules_to_not_convert and skip-module matching when composite-model module prefixes differ. + parent_config = config config = config.get_text_config() + # Propagate quantization_config from the composite parent config so that + # `get_hf_quantizer` can correctly detect the model as pre-quantized. + if hasattr(parent_config, "quantization_config"): + config.quantization_config = parent_config.quantization_config return model_class.from_pretrained( pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs ) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index deb1153d335e..078a910c368e 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -209,6 +209,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin): ("gpt_oss", "GptOssModel"), ("gptj", "GPTJModel"), ("granite", "GraniteModel"), + ("granite_speech", "GraniteSpeechForConditionalGeneration"), ("granitemoe", "GraniteMoeModel"), ("granitemoehybrid", "GraniteMoeHybridModel"), ("granitemoeshared", "GraniteMoeSharedModel"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index fd93e24edee1..6d0adc8473a6 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -704,8 +704,8 @@ def from_pretrained( else: tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None) - # if there is a config, we can check that the tokenizer class != than model class and can thus assume we need to use TokenizersBackend - # Skip this early exit if auto_map is present (custom tokenizer with trust_remote_code) + # if there is a config, we can check that the tokenizer class != than model class. + # Use the config class if it's a specialized tokenizer, otherwise fall back to TokenizersBackend. if ( tokenizer_auto_map is None and tokenizer_config_class is not None @@ -715,15 +715,20 @@ def from_pretrained( and (TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast")) != (tokenizer_config_class.removesuffix("Fast")) ): - # new model, but we ignore it unless the model type is the same + tokenizer_class = tokenizer_class_from_name(tokenizer_config_class) + if tokenizer_class is not None and tokenizer_class.__name__ not in ( + "TokenizersBackend", + "PythonBackend", + "PreTrainedTokenizerFast", + ): + return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + if TokenizersBackend is not None: - try: - return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) - except Exception as e: - logger.debug(f"Failed to use TokenizersBackend: {e}") + return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) - return tokenizer_class_from_name(tokenizer_config_class).from_pretrained( - pretrained_model_name_or_path, *inputs, **kwargs + raise ValueError( + f"Tokenizer class '{tokenizer_config_class}' specified in the tokenizer config was not found. " + f"The tokenizer may need to be converted or re-saved." ) if "_commit_hash" in tokenizer_config: diff --git a/src/transformers/models/conditional_detr/image_processing_pil_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_pil_conditional_detr.py index 359c4c706f7c..30740114d5f0 100644 --- a/src/transformers/models/conditional_detr/image_processing_pil_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_pil_conditional_detr.py @@ -61,6 +61,8 @@ logger = logging.get_logger(__name__) +SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) + class ConditionalDetrImageProcessorKwargs(ImagesKwargs, total=False): r""" @@ -76,9 +78,6 @@ class ConditionalDetrImageProcessorKwargs(ImagesKwargs, total=False): do_convert_annotations: bool -SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) - - # inspired by https://github.com/facebookresearch/conditional_detr/blob/master/datasets/coco.py#L33 def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndarray: """ diff --git a/src/transformers/models/conditional_detr/modular_conditional_detr.py b/src/transformers/models/conditional_detr/modular_conditional_detr.py index ffc1e78bee01..2205b85c5547 100644 --- a/src/transformers/models/conditional_detr/modular_conditional_detr.py +++ b/src/transformers/models/conditional_detr/modular_conditional_detr.py @@ -20,13 +20,12 @@ from ...image_transforms import ( center_to_corners_format, ) -from ...image_utils import AnnotationFormat from ...masking_utils import create_bidirectional_mask from ...modeling_outputs import ( BaseModelOutput, ) from ...modeling_utils import ALL_ATTENTION_FUNCTIONS -from ...processing_utils import ImagesKwargs, Unpack +from ...processing_utils import Unpack from ...utils import ( TensorType, TransformersKwargs, @@ -66,20 +65,6 @@ logger = logging.get_logger(__name__) -class ConditionalDetrImageProcessorKwargs(ImagesKwargs, total=False): - r""" - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the CONDITIONAL_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - """ - - format: str | AnnotationFormat - do_convert_annotations: bool - - class ConditionalDetrImageProcessor(DetrImageProcessor): def post_process_object_detection( self, outputs, threshold: float = 0.5, target_sizes: TensorType | list[tuple] = None, top_k: int = 100 diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index ab998cc99c21..fe3acd9aeddd 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -227,7 +227,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py index 3c62a564a31d..2bf7d347e85d 100644 --- a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py @@ -146,7 +146,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/deepseek_vl/modular_deepseek_vl.py b/src/transformers/models/deepseek_vl/modular_deepseek_vl.py index a56da6f3fe0a..be955c6fd41e 100644 --- a/src/transformers/models/deepseek_vl/modular_deepseek_vl.py +++ b/src/transformers/models/deepseek_vl/modular_deepseek_vl.py @@ -20,7 +20,7 @@ from ...configuration_utils import PreTrainedConfig from ...image_processing_utils import BatchFeature from ...image_utils import ImageInput -from ...processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, Unpack +from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack from ...tokenization_utils_base import ( PreTokenizedInput, TextInput, @@ -152,16 +152,6 @@ def generate(self): raise AttributeError("Not needed for DeepseekVL") -class DeepseekVLImageProcessorKwargs(ImagesKwargs, total=False): - r""" - min_size (`int`, *optional*, defaults to 14): - The minimum allowed size for the resized image. Ensures that neither the height nor width - falls below this value after resizing. - """ - - min_size: int - - class DeepseekVLImageProcessorPil(JanusImageProcessorPil): def postprocess(self): raise AttributeError("Not needed for DeepseekVL") diff --git a/src/transformers/models/deformable_detr/image_processing_pil_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_pil_deformable_detr.py index fcd95fa4647f..9c7ccc213910 100644 --- a/src/transformers/models/deformable_detr/image_processing_pil_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_pil_deformable_detr.py @@ -57,6 +57,8 @@ if is_torch_available(): import torch +SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) + class DeformableDetrImageProcessorKwargs(ImagesKwargs, total=False): r""" @@ -72,9 +74,6 @@ class DeformableDetrImageProcessorKwargs(ImagesKwargs, total=False): do_convert_annotations: bool -SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) - - # inspired by https://github.com/facebookresearch/deformable_detr/blob/master/datasets/coco.py#L33 def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndarray: """ diff --git a/src/transformers/models/deformable_detr/modular_deformable_detr.py b/src/transformers/models/deformable_detr/modular_deformable_detr.py index a2f80e8236ad..a4a5b4acd95a 100644 --- a/src/transformers/models/deformable_detr/modular_deformable_detr.py +++ b/src/transformers/models/deformable_detr/modular_deformable_detr.py @@ -23,11 +23,10 @@ from ... import initialization as init from ...backbone_utils import load_backbone from ...image_transforms import center_to_corners_format -from ...image_utils import AnnotationFormat from ...integrations import use_kernel_forward_from_hub from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...processing_utils import ImagesKwargs, Unpack +from ...processing_utils import Unpack from ...utils import ( ModelOutput, TensorType, @@ -61,20 +60,6 @@ logger = logging.get_logger(__name__) -class DeformableDetrImageProcessorKwargs(ImagesKwargs, total=False): - r""" - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the DEFORMABLE_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - """ - - format: str | AnnotationFormat - do_convert_annotations: bool - - class DeformableDetrImageProcessor(DetrImageProcessor): def post_process_object_detection( self, outputs, threshold: float = 0.5, target_sizes: TensorType | list[tuple] = None, top_k: int = 100 diff --git a/src/transformers/models/efficientloftr/modular_efficientloftr.py b/src/transformers/models/efficientloftr/modular_efficientloftr.py index 17e3e399a8df..86d8d34eba70 100644 --- a/src/transformers/models/efficientloftr/modular_efficientloftr.py +++ b/src/transformers/models/efficientloftr/modular_efficientloftr.py @@ -1,6 +1,5 @@ from typing import TYPE_CHECKING -from ...processing_utils import ImagesKwargs from ...utils import TensorType, is_torch_available from ...utils.import_utils import requires from ..superglue.image_processing_pil_superglue import SuperGlueImageProcessorPil @@ -14,15 +13,6 @@ from .modeling_efficientloftr import EfficientLoFTRKeypointMatchingOutput -class EfficientLoFTRImageProcessorKwargs(ImagesKwargs, total=False): - r""" - do_grayscale (`bool`, *optional*, defaults to `self.do_grayscale`): - Whether to convert the image to grayscale. Can be overridden by `do_grayscale` in the `preprocess` method. - """ - - do_grayscale: bool - - class EfficientLoFTRImageProcessor(SuperGlueImageProcessor): def post_process_keypoint_matching( self, diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 42bbb44b70a5..ad47bc0508a3 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -43,7 +43,7 @@ from ...modeling_outputs import BaseModelOutputWithPooling, MoeCausalLMOutputWithPast, MoeModelOutputWithPast from ...modeling_rope_utils import dynamic_rope_update from ...modeling_utils import PreTrainedModel -from ...processing_utils import ImagesKwargs, Unpack +from ...processing_utils import Unpack from ...utils import ( TensorType, TransformersKwargs, @@ -63,7 +63,7 @@ Ernie4_5_MoeStatics, Ernie4_5_MoeTopKRouter, ) -from ..glm4v.image_processing_glm4v import Glm4vImageProcessor +from ..glm4v.image_processing_glm4v import Glm4vImageProcessor, Glm4vImageProcessorKwargs from ..glm4v.image_processing_pil_glm4v import Glm4vImageProcessorPil from ..glm4v.modeling_glm4v import Glm4vForConditionalGeneration from ..mixtral.modeling_mixtral import load_balancing_loss_func @@ -1220,7 +1220,7 @@ def forward( ) -class Ernie4_5_VLMoeImageProcessorKwargs(ImagesKwargs, total=False): +class Ernie4_5_VLMoeImageProcessorKwargs(Glm4vImageProcessorKwargs): r""" patch_size (`int`, *optional*, defaults to 14): The spatial patch size of the vision encoder. @@ -1230,10 +1230,6 @@ class Ernie4_5_VLMoeImageProcessorKwargs(ImagesKwargs, total=False): The merge size of the vision encoder to llm encoder. """ - patch_size: int - temporal_patch_size: int - merge_size: int - class Ernie4_5_VLMoeImageProcessorPil(Glm4vImageProcessorPil): size = {"shortest_edge": 56 * 56, "longest_edge": 28 * 28 * 6177} diff --git a/src/transformers/models/exaone_moe/modeling_exaone_moe.py b/src/transformers/models/exaone_moe/modeling_exaone_moe.py index 2836a3c2245d..a7f80fc979c4 100644 --- a/src/transformers/models/exaone_moe/modeling_exaone_moe.py +++ b/src/transformers/models/exaone_moe/modeling_exaone_moe.py @@ -313,7 +313,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 9ebf8a5d1c07..3c07556708b0 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -1170,38 +1170,20 @@ def apply_rotary_pos_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, @use_kernelized_func(apply_rotary_pos_emb) class Gemma3nTextAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - def __init__(self, config: Gemma3nTextConfig, layer_idx: int): super().__init__() - self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None self.config = config self.layer_idx = layer_idx + self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None + self.is_sliding = self.layer_type == "sliding_attention" + self.sliding_window = config.sliding_window if self.is_sliding else None + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads self.scaling = 1.0 self.attention_dropout = self.config.attention_dropout self.is_causal = True - self.q_proj = nn.Linear( - config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias - ) - self.k_proj = nn.Linear( - config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.v_proj = nn.Linear( - config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias - ) - self.o_proj = nn.Linear( - config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias - ) - self.sliding_window = config.sliding_window if self.layer_type == "sliding_attention" else None - self.is_sliding = self.layer_type == "sliding_attention" - - self.q_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps) - self.k_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps) - self.v_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps, with_scale=False) - first_kv_shared_layer_idx = self.config.num_hidden_layers - self.config.num_kv_shared_layers self.is_kv_shared_layer = layer_idx >= first_kv_shared_layer_idx > 0 prev_layers = config.layer_types[:first_kv_shared_layer_idx] @@ -1216,14 +1198,35 @@ def __init__(self, config: Gemma3nTextConfig, layer_idx: int): config.layer_types[layer_idx] ) + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.q_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps) + + # Layers sharing kv states don't need any weight matrices + if not self.is_kv_shared_layer: + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.k_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps) + self.v_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps, with_scale=False) + + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + def forward( self, hidden_states: torch.Tensor, - position_embeddings: torch.Tensor = None, - attention_mask: torch.Tensor | None = None, + position_embeddings: torch.Tensor, + attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] hidden_shape = (*input_shape, -1, self.config.head_dim) @@ -1233,9 +1236,11 @@ def forward( query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) query_states = query_states.transpose(1, 2) - # For layers with shared KV (from kv sharing point onwards), we reuse the same keys/values states as the last non-sharing layer - if self.is_kv_shared_layer and past_key_values is not None: - key_states, value_states = past_key_values.shared_layers[self.kv_shared_layer_index] + # For layers with shared KV (from kv sharing point onwards), we reuse the same keys/values states as the last non-sharing layer. + # We cannot simply reuse the cached state if we have a Cache, as sliding layers will not remember the full states in their Cache + # once we are past the sliding window - so we always use `shared_kv_states` instead, even when past_key_values is not None + if self.is_kv_shared_layer: + key_states, value_states = shared_kv_states[self.kv_shared_layer_index] # Device of past layer may be different from current one key_states = key_states.to(query_states.device) value_states = value_states.to(query_states.device) @@ -1249,13 +1254,10 @@ def forward( value_states = self.v_norm(value_states) value_states = value_states.transpose(1, 2) - if past_key_values is not None: - if not self.is_kv_shared_layer: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) - if self.store_full_length_kv: - if not hasattr(past_key_values, "shared_layers"): - past_key_values.shared_layers = {} - past_key_values.shared_layers[self.layer_idx] = key_states, value_states + if past_key_values is not None and not self.is_kv_shared_layer: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + if self.store_full_length_kv: + shared_kv_states[self.layer_idx] = key_states, value_states attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -1305,6 +1307,7 @@ def forward( hidden_states: torch.Tensor, position_embeddings: torch.Tensor = None, per_layer_input: torch.Tensor = None, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, @@ -1319,6 +1322,7 @@ def forward( attn, _ = self.self_attn( hidden_states=active_prediction_normed, attention_mask=attention_mask, + shared_kv_states=shared_kv_states, position_ids=position_ids, position_embeddings=position_embeddings, past_key_values=past_key_values, @@ -1358,7 +1362,7 @@ class Gemma3nPreTrainedModel(PreTrainedModel): base_model_prefix = "model" supports_gradient_checkpointing = True _no_split_modules = ["Gemma3nTextDecoderLayer"] - _skip_keys_device_placement = ["past_key_values"] + _skip_keys_device_placement = ["past_key_values", "shared_kv_states"] _supports_flash_attn = True _supports_sdpa = True _supports_flex_attn = True @@ -1663,6 +1667,14 @@ def __init__(self, config: Gemma3nTextConfig): self.register_buffer("per_layer_projection_scale", torch.tensor(self.hidden_size**-0.5), persistent=False) self.register_buffer("per_layer_input_scale", torch.rsqrt(torch.tensor(2.0)), persistent=False) + # Update `_keys_to_ignore_on_load_unexpected` to drop all k/v proj and norms for the shared layers + self._keys_to_ignore_on_load_unexpected = [] + for i, layer in enumerate(self.layers): + if layer.self_attn.is_kv_shared_layer: + self._keys_to_ignore_on_load_unexpected.extend( + [f"layers.{i}.self_attn.{name}" for name in ("k_proj", "v_proj", "k_norm", "v_norm")] + ) + # Initialize weights and apply final processing self.post_init() @@ -1739,6 +1751,9 @@ def forward( for layer_type in self.config.layer_types: position_embeddings[layer_type] = self.rotary_emb(hidden_states, position_ids, layer_type) + # Initialize as empty dict - it will be filled in the right layers + shared_kv_states = {} + for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]): causal_mask = causal_mask_mapping[self.config.layer_types[i]] per_layer_input = per_layer_inputs[:, :, i, :] @@ -1747,6 +1762,7 @@ def forward( hidden_states, position_embeddings[self.config.layer_types[i]], per_layer_input, + shared_kv_states=shared_kv_states, attention_mask=causal_mask, position_ids=position_ids, past_key_values=past_key_values, @@ -1821,6 +1837,10 @@ def __init__(self, config: Gemma3nTextConfig): self.model = Gemma3nTextModel(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + # Grab the ones from the child + self._keys_to_ignore_on_load_unexpected = [ + f"model.{name}" for name in self.model._keys_to_ignore_on_load_unexpected + ] # Initialize weights and apply final processing self.post_init() @@ -1960,6 +1980,11 @@ def __init__(self, config: Gemma3nConfig): self.audio_tower = AutoModel.from_config(config.audio_config) self.embed_vision = Gemma3nMultimodalEmbedder(config.vision_config, config.text_config) self.embed_audio = Gemma3nMultimodalEmbedder(config.audio_config, config.text_config) + + # Grab the ones from the child + self._keys_to_ignore_on_load_unexpected = [ + f"language_model.{name}" for name in self.language_model._keys_to_ignore_on_load_unexpected + ] self.post_init() def get_input_embeddings(self): @@ -2214,6 +2239,10 @@ def __init__(self, config: Gemma3nConfig): super().__init__(config) self.model = Gemma3nModel(config) self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False) + # Grab the ones from the child + self._keys_to_ignore_on_load_unexpected = [ + f"model.{name}" for name in self.model._keys_to_ignore_on_load_unexpected + ] self.post_init() @auto_docstring diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 12146b7954e3..356baa483b10 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -26,6 +26,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...configuration_utils import PreTrainedConfig +from ...integrations import use_kernelized_func from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_outputs import BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS @@ -50,7 +51,6 @@ ) from ..gemma3.configuration_gemma3 import Gemma3TextConfig from ..gemma3.modeling_gemma3 import ( - Gemma3Attention, Gemma3DecoderLayer, Gemma3ForCausalLM, Gemma3RotaryEmbedding, @@ -1463,13 +1463,21 @@ def apply_rotary_pos_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, return (x * cos) + (rotate_half(x) * sin) -class Gemma3nTextAttention(Gemma3Attention): +@use_kernelized_func(apply_rotary_pos_emb) +class Gemma3nTextAttention(nn.Module): def __init__(self, config: Gemma3nTextConfig, layer_idx: int): - super().__init__(config, layer_idx) - self.is_causal = True - del self.attn_logit_softcapping + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.layer_type = config.layer_types[layer_idx] if hasattr(config, "layer_types") else None + self.is_sliding = self.layer_type == "sliding_attention" + self.sliding_window = config.sliding_window if self.is_sliding else None + + self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) + self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads self.scaling = 1.0 - self.v_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps, with_scale=False) + self.attention_dropout = self.config.attention_dropout + self.is_causal = True first_kv_shared_layer_idx = self.config.num_hidden_layers - self.config.num_kv_shared_layers self.is_kv_shared_layer = layer_idx >= first_kv_shared_layer_idx > 0 @@ -1485,14 +1493,35 @@ def __init__(self, config: Gemma3nTextConfig, layer_idx: int): config.layer_types[layer_idx] ) + self.q_proj = nn.Linear( + config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias + ) + self.q_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps) + + # Layers sharing kv states don't need any weight matrices + if not self.is_kv_shared_layer: + self.k_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.v_proj = nn.Linear( + config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias + ) + self.k_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps) + self.v_norm = Gemma3nRMSNorm(dim=config.head_dim, eps=config.rms_norm_eps, with_scale=False) + + self.o_proj = nn.Linear( + config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias + ) + def forward( self, hidden_states: torch.Tensor, - position_embeddings: torch.Tensor = None, - attention_mask: torch.Tensor | None = None, + position_embeddings: torch.Tensor, + attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, **kwargs: Unpack[TransformersKwargs], - ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] hidden_shape = (*input_shape, -1, self.config.head_dim) @@ -1502,9 +1531,11 @@ def forward( query_states = apply_rotary_pos_emb(query_states, cos, sin, unsqueeze_dim=2) query_states = query_states.transpose(1, 2) - # For layers with shared KV (from kv sharing point onwards), we reuse the same keys/values states as the last non-sharing layer - if self.is_kv_shared_layer and past_key_values is not None: - key_states, value_states = past_key_values.shared_layers[self.kv_shared_layer_index] + # For layers with shared KV (from kv sharing point onwards), we reuse the same keys/values states as the last non-sharing layer. + # We cannot simply reuse the cached state if we have a Cache, as sliding layers will not remember the full states in their Cache + # once we are past the sliding window - so we always use `shared_kv_states` instead, even when past_key_values is not None + if self.is_kv_shared_layer: + key_states, value_states = shared_kv_states[self.kv_shared_layer_index] # Device of past layer may be different from current one key_states = key_states.to(query_states.device) value_states = value_states.to(query_states.device) @@ -1518,13 +1549,10 @@ def forward( value_states = self.v_norm(value_states) value_states = value_states.transpose(1, 2) - if past_key_values is not None: - if not self.is_kv_shared_layer: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) - if self.store_full_length_kv: - if not hasattr(past_key_values, "shared_layers"): - past_key_values.shared_layers = {} - past_key_values.shared_layers[self.layer_idx] = key_states, value_states + if past_key_values is not None and not self.is_kv_shared_layer: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + if self.store_full_length_kv: + shared_kv_states[self.layer_idx] = key_states, value_states attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -1567,6 +1595,7 @@ def forward( hidden_states: torch.Tensor, position_embeddings: torch.Tensor = None, per_layer_input: torch.Tensor = None, + shared_kv_states: dict[int, tuple[torch.Tensor, torch.Tensor]] | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, @@ -1581,6 +1610,7 @@ def forward( attn, _ = self.self_attn( hidden_states=active_prediction_normed, attention_mask=attention_mask, + shared_kv_states=shared_kv_states, position_ids=position_ids, position_embeddings=position_embeddings, past_key_values=past_key_values, @@ -1617,6 +1647,7 @@ def forward( class Gemma3nPreTrainedModel(Gemma2PreTrainedModel): config: Gemma3nConfig input_modalities = ("image", "text", "audio") + _skip_keys_device_placement = ["past_key_values", "shared_kv_states"] _no_split_modules = ["Gemma3nTextDecoderLayer"] _can_record_outputs = { "hidden_states": Gemma3nTextDecoderLayer, @@ -1828,6 +1859,14 @@ def __init__(self, config: Gemma3nTextConfig): self.register_buffer("per_layer_projection_scale", torch.tensor(self.hidden_size**-0.5), persistent=False) self.register_buffer("per_layer_input_scale", torch.rsqrt(torch.tensor(2.0)), persistent=False) + # Update `_keys_to_ignore_on_load_unexpected` to drop all k/v proj and norms for the shared layers + self._keys_to_ignore_on_load_unexpected = [] + for i, layer in enumerate(self.layers): + if layer.self_attn.is_kv_shared_layer: + self._keys_to_ignore_on_load_unexpected.extend( + [f"layers.{i}.self_attn.{name}" for name in ("k_proj", "v_proj", "k_norm", "v_norm")] + ) + def get_per_layer_inputs(self, input_ids: torch.LongTensor) -> torch.Tensor: return self.embed_tokens_per_layer(input_ids).reshape( *input_ids.shape, @@ -1936,6 +1975,9 @@ def forward( for layer_type in self.config.layer_types: position_embeddings[layer_type] = self.rotary_emb(hidden_states, position_ids, layer_type) + # Initialize as empty dict - it will be filled in the right layers + shared_kv_states = {} + for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]): causal_mask = causal_mask_mapping[self.config.layer_types[i]] per_layer_input = per_layer_inputs[:, :, i, :] @@ -1944,6 +1986,7 @@ def forward( hidden_states, position_embeddings[self.config.layer_types[i]], per_layer_input, + shared_kv_states=shared_kv_states, attention_mask=causal_mask, position_ids=position_ids, past_key_values=past_key_values, @@ -1974,7 +2017,12 @@ def forward( @auto_docstring(custom_intro="The base Gemma 3n language model with a language modeling head.") class Gemma3nForCausalLM(Gemma3ForCausalLM): - pass + def __init__(self, config: Gemma3nTextConfig): + super().__init__(config) + # Grab the ones from the child + self._keys_to_ignore_on_load_unexpected = [ + f"model.{name}" for name in self.model._keys_to_ignore_on_load_unexpected + ] class Gemma3nMultimodalEmbedder(nn.Module): @@ -2043,6 +2091,11 @@ def __init__(self, config: Gemma3nConfig): self.embed_vision = Gemma3nMultimodalEmbedder(config.vision_config, config.text_config) self.embed_audio = Gemma3nMultimodalEmbedder(config.audio_config, config.text_config) + # Grab the ones from the child + self._keys_to_ignore_on_load_unexpected = [ + f"language_model.{name}" for name in self.language_model._keys_to_ignore_on_load_unexpected + ] + def get_per_layer_input_embeddings(self): return self.language_model.embed_tokens_per_layer @@ -2284,6 +2337,13 @@ def get_audio_features( class Gemma3nForConditionalGeneration(PaliGemmaForConditionalGeneration): accepts_loss_kwargs = False + def __init__(self, config: Gemma3nConfig): + super().__init__(config) + # Grab the ones from the child + self._keys_to_ignore_on_load_unexpected = [ + f"model.{name}" for name in self.model._keys_to_ignore_on_load_unexpected + ] + def get_per_layer_input_embeddings(self): return self.model.get_per_layer_input_embeddings() diff --git a/src/transformers/models/gemma4/configuration_gemma4.py b/src/transformers/models/gemma4/configuration_gemma4.py index 7ae940d861dd..fe38dc1739ab 100644 --- a/src/transformers/models/gemma4/configuration_gemma4.py +++ b/src/transformers/models/gemma4/configuration_gemma4.py @@ -138,6 +138,16 @@ class Gemma4TextConfig(PreTrainedConfig): "layers.*.experts.down_proj": "rowwise", "layers.*.experts": "moe_tp_experts", } + base_model_ep_plan = { + # EP plan for google/gemma-4-26B-A4B-it: do not tp in attention (num_global_key_value_heads=2 too small to partition) + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + "layers.*.router": "ep_router", + "layers.*.experts.gate_up_proj": "grouped_gemm", + "layers.*.experts.down_proj": "grouped_gemm", + "layers.*.experts": "moe_tp_experts", + } base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), diff --git a/src/transformers/models/gemma4/convert_gemma4_weights.py b/src/transformers/models/gemma4/convert_gemma4_weights.py index d370793288be..53940445c7e6 100644 --- a/src/transformers/models/gemma4/convert_gemma4_weights.py +++ b/src/transformers/models/gemma4/convert_gemma4_weights.py @@ -712,6 +712,7 @@ def convert_transformer_weights( converted_paths: list[str] = [] converted_weights: list[Any] = [] + first_kv_shared_layer_idx = config.num_hidden_layers - getattr(config, "num_kv_shared_layers", 0) # Handle new checkpoint format: transformer/layer_N/... # TODO(philculliton):Direct handling for unstacked checkpoint type, needs to be merged to allow for unified tensor handling @@ -719,6 +720,7 @@ def convert_transformer_weights( # Extract layer number from path like "transformer/layer_0/attn/q_einsum" layer_str = path.split("/")[1] # "layer_0" layer_idx = int(layer_str.replace("layer_", "")) # 0 + is_kv_shared_layer = layer_idx >= first_kv_shared_layer_idx > 0 base_path = f"layers.{layer_idx}" # Determine head_dim from actual checkpoint weight dimensions @@ -744,7 +746,7 @@ def convert_transformer_weights( converted_weights.append( matrix.transpose(2, 0, 1).reshape(config.hidden_size, config.num_attention_heads * head_dim) ) - elif path.endswith("attn/kv_einsum"): + elif path.endswith("attn/kv_einsum") and not is_kv_shared_layer: converted_paths.extend( [ f"{base_path}.self_attn.k_proj.weight", @@ -759,7 +761,7 @@ def convert_transformer_weights( v_proj_weights.reshape(kv_proj_shape).transpose(), ] ) - elif path.endswith("attn/k_einsum"): + elif path.endswith("attn/k_einsum") and not is_kv_shared_layer: converted_paths.append(f"{base_path}.self_attn.k_proj.weight") converted_weights.append( matrix.transpose(1, 0, 2) @@ -776,7 +778,7 @@ def convert_transformer_weights( elif path.endswith("attn/query_norm"): converted_paths.append(f"{base_path}.self_attn.q_norm.weight") converted_weights.append(matrix) - elif path.endswith("attn/key_norm"): + elif path.endswith("attn/key_norm") and not is_kv_shared_layer: converted_paths.append(f"{base_path}.self_attn.k_norm.weight") converted_weights.append(matrix) elif path.endswith("mlp/gating_einsum"): @@ -822,6 +824,7 @@ def convert_transformer_weights( for i, matrix in enumerate(weights): layer_idx = _SLIDING_WINDOW_PATTERN * i + attention_type_index + is_kv_shared_layer = layer_idx >= first_kv_shared_layer_idx > 0 base_path = f"layers.{layer_idx}" head_dim = ( config.global_head_dim @@ -837,7 +840,7 @@ def convert_transformer_weights( converted_weights.append( matrix.transpose(2, 0, 1).reshape(config.hidden_size, config.num_attention_heads * head_dim) ) - elif path.endswith("attn/kv_einsum"): + elif path.endswith("attn/kv_einsum") and not is_kv_shared_layer: converted_paths.extend( [ f"{base_path}.self_attn.k_proj.weight", @@ -852,7 +855,7 @@ def convert_transformer_weights( v_proj_weights.reshape(kv_proj_shape).transpose(), ] ) - elif path.endswith("attn/k_einsum"): + elif path.endswith("attn/k_einsum") and not is_kv_shared_layer: converted_paths.append(f"{base_path}.self_attn.k_proj.weight") converted_weights.append( matrix.transpose(1, 0, 2) @@ -869,7 +872,7 @@ def convert_transformer_weights( elif path.endswith("attn/query_norm"): converted_paths.append(f"{base_path}.self_attn.q_norm.weight") converted_weights.append(matrix) - elif path.endswith("attn/key_norm"): + elif path.endswith("attn/key_norm") and not is_kv_shared_layer: converted_paths.append(f"{base_path}.self_attn.k_norm.weight") converted_weights.append(matrix) elif path.endswith("mlp/gating_einsum"): diff --git a/src/transformers/models/gemma4/modeling_gemma4.py b/src/transformers/models/gemma4/modeling_gemma4.py index 88c340a9414b..5b147f95ba36 100644 --- a/src/transformers/models/gemma4/modeling_gemma4.py +++ b/src/transformers/models/gemma4/modeling_gemma4.py @@ -2151,23 +2151,24 @@ class Gemma4Model(Gemma4PreTrainedModel): def __init__(self, config: Gemma4Config): super().__init__(config) + self.vision_tower = AutoModel.from_config(config.vision_config) if config.vision_config is not None else None self.vocab_size = config.text_config.vocab_size language_model = AutoModel.from_config(config=config.text_config) self.language_model = language_model self.vocab_size_per_layer_input = config.text_config.vocab_size_per_layer_input - self.vision_tower = AutoModel.from_config(config.vision_config) if config.vision_config is not None else None + self.audio_tower = AutoModel.from_config(config.audio_config) if config.audio_config is not None else None self.embed_vision = ( Gemma4MultimodalEmbedder(config.vision_config, config.text_config) if config.vision_config is not None else None ) - self.audio_tower = AutoModel.from_config(config.audio_config) if config.audio_config is not None else None self.embed_audio = ( Gemma4MultimodalEmbedder(config.audio_config, config.text_config) if config.audio_config is not None else None ) + # Grab the ones from the child self._keys_to_ignore_on_load_unexpected = [ f"language_model.{name}" for name in self.language_model._keys_to_ignore_on_load_unexpected diff --git a/src/transformers/models/gemma4/modular_gemma4.py b/src/transformers/models/gemma4/modular_gemma4.py index 0cddf103f3bf..12412b319b5c 100644 --- a/src/transformers/models/gemma4/modular_gemma4.py +++ b/src/transformers/models/gemma4/modular_gemma4.py @@ -1162,7 +1162,6 @@ class Gemma4PreTrainedModel(Gemma3nPreTrainedModel): _no_split_modules = ["Gemma4TextDecoderLayer", "Gemma4VisionEncoderLayer", "Gemma4AudioLayer"] input_modalities = ("image", "text", "video", "audio") _can_record_outputs = None # override - _skip_keys_device_placement = ["past_key_values", "shared_kv_states"] @torch.no_grad() def _init_weights(self, module): @@ -1723,26 +1722,18 @@ def create_causal_mask_mapping( class Gemma4Model(Gemma3nModel): def __init__(self, config: Gemma4Config): super().__init__(config) - del self.vision_tower - del self.embed_vision self.vision_tower = AutoModel.from_config(config.vision_config) if config.vision_config is not None else None self.embed_vision = ( Gemma4MultimodalEmbedder(config.vision_config, config.text_config) if config.vision_config is not None else None ) - del self.audio_tower - del self.embed_audio self.audio_tower = AutoModel.from_config(config.audio_config) if config.audio_config is not None else None self.embed_audio = ( Gemma4MultimodalEmbedder(config.audio_config, config.text_config) if config.audio_config is not None else None ) - # Grab the ones from the child - self._keys_to_ignore_on_load_unexpected = [ - f"language_model.{name}" for name in self.language_model._keys_to_ignore_on_load_unexpected - ] def get_per_layer_input_embeddings(self): return self.language_model.embed_tokens_per_layer @@ -2034,13 +2025,6 @@ def get_audio_features( class Gemma4ForConditionalGeneration(Gemma3nForConditionalGeneration): base_model_prefix = "model" - def __init__(self, config: Gemma4Config): - super().__init__(config) - # Grab the ones from the child - self._keys_to_ignore_on_load_unexpected = [ - f"model.{name}" for name in self.model._keys_to_ignore_on_load_unexpected - ] - def get_per_layer_input_embeddings(self): return self.model.get_per_layer_input_embeddings() diff --git a/src/transformers/models/glm4_moe/modeling_glm4_moe.py b/src/transformers/models/glm4_moe/modeling_glm4_moe.py index 1bc20c8322d9..cc5a564ab86f 100644 --- a/src/transformers/models/glm4_moe/modeling_glm4_moe.py +++ b/src/transformers/models/glm4_moe/modeling_glm4_moe.py @@ -402,7 +402,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py index d59fd2ab996e..0b8ccc865775 100644 --- a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py +++ b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py @@ -477,7 +477,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index b3f5118a3d67..3bf3dc157d3f 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -292,7 +292,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/glm_image/image_processing_pil_glm_image.py b/src/transformers/models/glm_image/image_processing_pil_glm_image.py index 2dde18ef2066..355bb04adb67 100644 --- a/src/transformers/models/glm_image/image_processing_pil_glm_image.py +++ b/src/transformers/models/glm_image/image_processing_pil_glm_image.py @@ -30,7 +30,6 @@ from ...utils import TensorType, auto_docstring -# Adapted from transformers.models.glm_image.image_processing_glm_image.GlmImageImageProcessorKwargs class GlmImageImageProcessorKwargs(ImagesKwargs, total=False): r""" min_pixels (`int`, *optional*, defaults to `56 * 56`): diff --git a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py index 950deba0800e..4fa6930ea518 100644 --- a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py +++ b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py @@ -560,7 +560,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index aff96cad3217..9430e8a91018 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -356,6 +356,7 @@ def forward(self, audio_features): ) class GlmAsrForConditionalGeneration(GlmAsrPreTrainedModel, GenerationMixin): _keep_in_fp32_modules_strict = None + _supports_attention_backend = True _tp_plan = None _pp_plan = None diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index ff0b8b6062a4..2c6085eb3a18 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -357,6 +357,8 @@ def __init__(self, config: GlmAsrConfig): """ ) class GlmAsrForConditionalGeneration(AudioFlamingo3ForConditionalGeneration): + _supports_attention_backend = True + @can_return_tuple @auto_docstring( custom_intro="Compute audio embeddings from log-mel input features using the audio encoder and multi-modal projector." diff --git a/src/transformers/models/gpt_oss/configuration_gpt_oss.py b/src/transformers/models/gpt_oss/configuration_gpt_oss.py index b745c8f0f63d..c0a5ea4f21c5 100644 --- a/src/transformers/models/gpt_oss/configuration_gpt_oss.py +++ b/src/transformers/models/gpt_oss/configuration_gpt_oss.py @@ -23,6 +23,9 @@ @strict class GptOssConfig(PreTrainedConfig): model_type = "gpt_oss" + attribute_map = { + "num_experts": "num_local_experts", + } default_theta = 150000.0 base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), diff --git a/src/transformers/models/granite_speech/modeling_granite_speech.py b/src/transformers/models/granite_speech/modeling_granite_speech.py index 0fbc1d1035bf..03024afe8337 100644 --- a/src/transformers/models/granite_speech/modeling_granite_speech.py +++ b/src/transformers/models/granite_speech/modeling_granite_speech.py @@ -327,6 +327,8 @@ def forward( """ ) class GraniteSpeechForConditionalGeneration(GraniteSpeechPreTrainedModel, GenerationMixin): + _supports_attention_backend = True + def __init__(self, config: GraniteSpeechConfig): super().__init__(config) # NOTE: It doesn't matter when we initialize from config, but we should be careful diff --git a/src/transformers/models/grounding_dino/image_processing_pil_grounding_dino.py b/src/transformers/models/grounding_dino/image_processing_pil_grounding_dino.py index 31c59e5f3930..c95d7cb386bd 100644 --- a/src/transformers/models/grounding_dino/image_processing_pil_grounding_dino.py +++ b/src/transformers/models/grounding_dino/image_processing_pil_grounding_dino.py @@ -67,6 +67,8 @@ if is_torch_available(): import torch +SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) + class GroundingDinoImageProcessorKwargs(ImagesKwargs, total=False): r""" @@ -82,9 +84,6 @@ class GroundingDinoImageProcessorKwargs(ImagesKwargs, total=False): do_convert_annotations: bool -SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) - - # inspired by https://github.com/facebookresearch/grounding_dino/blob/master/datasets/coco.py#L33 def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndarray: """ diff --git a/src/transformers/models/grounding_dino/modular_grounding_dino.py b/src/transformers/models/grounding_dino/modular_grounding_dino.py index 483ad262a602..bd35fd512ffe 100644 --- a/src/transformers/models/grounding_dino/modular_grounding_dino.py +++ b/src/transformers/models/grounding_dino/modular_grounding_dino.py @@ -25,8 +25,6 @@ from transformers.models.detr.image_processing_pil_detr import DetrImageProcessorPil from ...image_transforms import center_to_corners_format -from ...image_utils import AnnotationFormat -from ...processing_utils import ImagesKwargs from ...utils import ( TensorType, logging, @@ -70,20 +68,6 @@ def _scale_boxes(boxes, target_sizes): return boxes -class GroundingDinoImageProcessorKwargs(ImagesKwargs, total=False): - r""" - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the GROUNDING_DINO model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - """ - - format: str | AnnotationFormat - do_convert_annotations: bool - - class GroundingDinoImageProcessor(DetrImageProcessor): def post_process_object_detection( self, diff --git a/src/transformers/models/lightglue/modular_lightglue.py b/src/transformers/models/lightglue/modular_lightglue.py index 62082b678b00..afc8a3efec25 100644 --- a/src/transformers/models/lightglue/modular_lightglue.py +++ b/src/transformers/models/lightglue/modular_lightglue.py @@ -23,7 +23,7 @@ from ...configuration_utils import PreTrainedConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import ImagesKwargs, Unpack +from ...processing_utils import Unpack from ...utils import ModelOutput, TensorType, auto_docstring, can_return_tuple, logging from ...utils.import_utils import requires from ..auto import CONFIG_MAPPING, AutoConfig @@ -32,7 +32,7 @@ from ..cohere.modeling_cohere import apply_rotary_pos_emb from ..llama.modeling_llama import LlamaAttention, eager_attention_forward from ..superglue.image_processing_pil_superglue import SuperGlueImageProcessorPil -from ..superglue.image_processing_superglue import SuperGlueImageProcessor +from ..superglue.image_processing_superglue import SuperGlueImageProcessor, SuperGlueImageProcessorKwargs from ..superpoint import SuperPointConfig @@ -154,13 +154,8 @@ class LightGlueKeypointMatchingOutput(ModelOutput): attentions: tuple[torch.FloatTensor] | None = None -class LightGlueImageProcessorKwargs(ImagesKwargs, total=False): - r""" - do_grayscale (`bool`, *optional*, defaults to `self.do_grayscale`): - Whether to convert the image to grayscale. Can be overridden by `do_grayscale` in the `preprocess` method. - """ - - do_grayscale: bool +class LightGlueImageProcessorKwargs(SuperGlueImageProcessorKwargs): + pass class LightGlueImageProcessor(SuperGlueImageProcessor): diff --git a/src/transformers/models/llava_onevision/image_processing_pil_llava_onevision.py b/src/transformers/models/llava_onevision/image_processing_pil_llava_onevision.py index f70a5c124bd7..23534a65d70f 100644 --- a/src/transformers/models/llava_onevision/image_processing_pil_llava_onevision.py +++ b/src/transformers/models/llava_onevision/image_processing_pil_llava_onevision.py @@ -36,7 +36,6 @@ from ...utils import TensorType, auto_docstring -# Adapted from transformers.models.llava_onevision.image_processing_llava_onevision.LlavaOnevisionImageProcessorKwargs class LlavaOnevisionImageProcessorKwargs(ImagesKwargs, total=False): r""" image_grid_pinpoints (`list[list[int]]`, *optional*): diff --git a/src/transformers/models/mask2former/modular_mask2former.py b/src/transformers/models/mask2former/modular_mask2former.py index 87f2b834991f..089baffe5df7 100644 --- a/src/transformers/models/mask2former/modular_mask2former.py +++ b/src/transformers/models/mask2former/modular_mask2former.py @@ -15,8 +15,6 @@ import torch from torch import nn -from ...image_utils import SizeDict -from ...processing_utils import ImagesKwargs from ...utils import ( TensorType, logging, @@ -35,32 +33,6 @@ logger = logging.get_logger(__name__) -class Mask2FormerImageProcessorKwargs(ImagesKwargs, total=False): - r""" - ignore_index (`int`, *optional*): - Label to be assigned to background pixels in segmentation maps. If provided, segmentation map pixels - denoted with 0 (background) will be replaced with `ignore_index`. - do_reduce_labels (`bool`, *optional*, defaults to `False`): - Whether or not to decrement all label values of segmentation maps by 1. Usually used for datasets where 0 - is used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). - The background label will be replaced by `ignore_index`. - num_labels (`int`, *optional*): - The number of labels in the segmentation map. - size_divisor (`int`, *optional*, defaults to `32`): - Some backbones need images divisible by a certain number. If not passed, it defaults to the value used in - Swin Transformer. - pad_size (`SizeDict`, *optional*): - The size to pad the images to. Must be larger than any image size provided for preprocessing. If `pad_size` - is not provided, images will be padded to the largest height and width in the batch. - """ - - ignore_index: int | None - do_reduce_labels: bool - num_labels: int | None - size_divisor: int - pad_size: SizeDict | None - - class Mask2FormerImageProcessor(MaskFormerImageProcessor): def post_process_semantic_segmentation( self, outputs, target_sizes: list[tuple[int, int]] | None = None diff --git a/src/transformers/models/musicflamingo/modeling_musicflamingo.py b/src/transformers/models/musicflamingo/modeling_musicflamingo.py index adec95bbf3e1..4ec4215a2989 100644 --- a/src/transformers/models/musicflamingo/modeling_musicflamingo.py +++ b/src/transformers/models/musicflamingo/modeling_musicflamingo.py @@ -200,6 +200,7 @@ def apply_rotary_time_emb(hidden_states, cos, sin): ) class MusicFlamingoForConditionalGeneration(MusicFlamingoPreTrainedModel, GenerationMixin): _keep_in_fp32_modules_strict = None + _supports_attention_backend = True _tp_plan = None _pp_plan = None diff --git a/src/transformers/models/nemotron_h/configuration_nemotron_h.py b/src/transformers/models/nemotron_h/configuration_nemotron_h.py index 4d9361f1e5d2..1c39584c45ac 100644 --- a/src/transformers/models/nemotron_h/configuration_nemotron_h.py +++ b/src/transformers/models/nemotron_h/configuration_nemotron_h.py @@ -27,7 +27,7 @@ class NemotronHConfig(PreTrainedConfig): r""" layers_block_type (`list`, *optional*): - Explicit list of layer types for each layer. Each element must be one of: "mamba", "attention", or "moe". + Explicit list of layer types for each layer. Each element must be one of: "mlp", "mamba", "attention", or "moe". The number of layers is determined by the length of this list. num_logits_to_keep (`int`, *optional*, defaults to 1): Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. @@ -158,20 +158,20 @@ def __post_init__(self, **kwargs): # Always pop hybrid_override_pattern from kwargs to prevent it from being set as an attribute if "hybrid_override_pattern" in kwargs: pattern = kwargs.pop("hybrid_override_pattern") - if self.layers_block_type is None: - self.layers_block_type = self._pattern_to_list(pattern) - elif self.layers_block_type is None: + if self.layer_types is None: + self.layer_types = self._pattern_to_list(pattern) + elif self.layer_types is None: # Default layers_block_type if not provided - self.layers_block_type = ["mamba", "moe", "attention", "moe"] + self.layer_types = ["mamba", "moe", "attention", "mlp"] # Note: num_hidden_layers is deprecated and ignored if layers_block_type is explicitly provided # It's only kept for backward compatibility when loading old configs if self.num_hidden_layers is not None: # Warn if num_hidden_layers is provided but doesn't match layers_block_type - if len(self.layers_block_type) != self.num_hidden_layers: + if len(self.layer_types) != self.num_hidden_layers: logger.warning( f"num_hidden_layers ({self.num_hidden_layers}) is deprecated and doesn't match " - f"layers_block_type length ({len(self.layers_block_type)}). Using layers_block_type length." + f"layer_types length ({len(self.layer_types)}). Using layers_block_type length." ) # Backward compatibility: convert mtp_hybrid_override_pattern to mtp_layers_block_type @@ -191,18 +191,16 @@ def __post_init__(self, **kwargs): super().__post_init__(**kwargs) @staticmethod - def validate_layers_block_type(self): + def validate_layer_type(self): """ Validate layers_block_type list. """ - if not isinstance(self.layers_block_type, list): - raise ValueError( - f"`layers_block_type` must be a list of strings. Got type: {type(self.layers_block_type)}" - ) - - valid_types = {"mamba", "attention", "moe"} - if not all(block_type in valid_types for block_type in self.layers_block_type): - invalid = set(self.layers_block_type) - valid_types + if not isinstance(self.layer_types, list): + raise ValueError(f"`layers_block_type` must be a list of strings. Got type: {type(self.layer_types)}") + + valid_types = {"mamba", "attention", "moe", "mlp"} + if not all(block_type in valid_types for block_type in self.layer_types): + invalid = set(self.layer_types) - valid_types raise ValueError(f"`layers_block_type` contains invalid types: {invalid}. Must be one of: {valid_types}") if self.num_nextn_predict_layers > 0: @@ -218,7 +216,6 @@ def validate_layers_block_type(self): f"`mtp_layers_block_type` must be a list of strings. Got type: {type(self.mtp_layers_block_type)}" ) - valid_types = {"mamba", "attention", "moe"} if not all(block_type in valid_types for block_type in self.mtp_layers_block_type): invalid = set(self.mtp_layers_block_type) - valid_types raise ValueError( @@ -261,13 +258,13 @@ def mtp_hybrid_override_pattern(self) -> str: @staticmethod def _list_to_pattern(layers_list: list) -> str: """Convert list of layer types back to pattern string (for backward compatibility).""" - reverse_mapping = {"mamba": "M", "moe": "E", "attention": "*"} + reverse_mapping = {"mamba": "M", "moe": "E", "attention": "*", "mlp": "-"} return "".join(reverse_mapping[layer_type] for layer_type in layers_list) @staticmethod def _pattern_to_list(pattern: str) -> list: """Convert pattern string to list of layer types (for backward compatibility).""" - pattern_mapping = {"M": "mamba", "E": "moe", "*": "attention"} + pattern_mapping = {"M": "mamba", "E": "moe", "*": "attention", "-": "mlp"} return [pattern_mapping[char] for char in pattern] diff --git a/src/transformers/models/nemotron_h/modeling_nemotron_h.py b/src/transformers/models/nemotron_h/modeling_nemotron_h.py index a32c349ce18f..6af7fd477564 100644 --- a/src/transformers/models/nemotron_h/modeling_nemotron_h.py +++ b/src/transformers/models/nemotron_h/modeling_nemotron_h.py @@ -594,7 +594,7 @@ def extra_repr(self): class NemotronHMLP(nn.Module): - def __init__(self, config, intermediate_size=None): + def __init__(self, config, intermediate_size=None, **kwargs): super().__init__() self.config = config self.hidden_size = config.hidden_size @@ -719,7 +719,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: @@ -889,6 +889,7 @@ def forward( "mamba": NemotronHMamba2Mixer, "attention": NemotronHAttention, "moe": NemotronHMoE, + "mlp": NemotronHMLP, } @@ -1081,6 +1082,7 @@ def forward( "mamba": mamba_mask, "attention": causal_mask, "moe": None, + "mlp": None, } for layer_idx, mixer_block in enumerate(self.layers): diff --git a/src/transformers/models/nemotron_h/modular_nemotron_h.py b/src/transformers/models/nemotron_h/modular_nemotron_h.py index a7433a982f1c..f49597f43140 100644 --- a/src/transformers/models/nemotron_h/modular_nemotron_h.py +++ b/src/transformers/models/nemotron_h/modular_nemotron_h.py @@ -106,8 +106,8 @@ class NemotronHRMSNorm(LlamaRMSNorm): pass -class NemotronHMLP(NemotronMLP): - def __init__(self, config, intermediate_size=None): +class NemotronHMLP(NemotronMLP, nn.Module): + def __init__(self, config, intermediate_size=None, **kwargs): nn.Module.__init__() self.config = config self.hidden_size = config.hidden_size @@ -242,6 +242,7 @@ def forward( "mamba": NemotronHMamba2Mixer, "attention": NemotronHAttention, "moe": NemotronHMoE, + "mlp": NemotronHMLP, } @@ -434,6 +435,7 @@ def forward( "mamba": mamba_mask, "attention": causal_mask, "moe": None, + "mlp": None, } for layer_idx, mixer_block in enumerate(self.layers): diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 20a897059a4e..02895d6e2576 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -38,9 +38,8 @@ from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPast, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...models.qwen2_vl.image_processing_pil_qwen2_vl import Qwen2VLImageProcessorPil -from ...models.qwen2_vl.image_processing_qwen2_vl import Qwen2VLImageProcessor +from ...models.qwen2_vl.image_processing_qwen2_vl import Qwen2VLImageProcessor, Qwen2VLImageProcessorKwargs from ...processing_utils import ( - ImagesKwargs, ProcessingKwargs, ProcessorMixin, Unpack, @@ -123,7 +122,7 @@ def smart_resize( return h_bar, w_bar -class PaddleOCRVLImageProcessorKwargs(ImagesKwargs, total=False): +class PaddleOCRVLImageProcessorKwargs(Qwen2VLImageProcessorKwargs): r""" patch_size (`int`, *optional*, defaults to 14): The spatial patch size of the vision encoder. @@ -133,12 +132,6 @@ class PaddleOCRVLImageProcessorKwargs(ImagesKwargs, total=False): The merge size of the vision encoder to llm encoder. """ - min_pixels: int - max_pixels: int - patch_size: int - temporal_patch_size: int - merge_size: int - class PaddleOCRVLImageProcessorPil(Qwen2VLImageProcessorPil): size = {"shortest_edge": 384 * 384, "longest_edge": 1536 * 1536} diff --git a/src/transformers/models/rt_detr/image_processing_pil_rt_detr.py b/src/transformers/models/rt_detr/image_processing_pil_rt_detr.py index 1fe55d067653..669843e9f949 100644 --- a/src/transformers/models/rt_detr/image_processing_pil_rt_detr.py +++ b/src/transformers/models/rt_detr/image_processing_pil_rt_detr.py @@ -54,6 +54,8 @@ if is_torch_available(): import torch +SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) + class RTDetrImageProcessorKwargs(ImagesKwargs, total=False): r""" @@ -69,9 +71,6 @@ class RTDetrImageProcessorKwargs(ImagesKwargs, total=False): do_convert_annotations: bool -SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) - - def prepare_coco_detection_annotation_pil( image, target, diff --git a/src/transformers/models/rt_detr/modular_rt_detr.py b/src/transformers/models/rt_detr/modular_rt_detr.py index 97136541d6ec..cd4e8faf3fc2 100644 --- a/src/transformers/models/rt_detr/modular_rt_detr.py +++ b/src/transformers/models/rt_detr/modular_rt_detr.py @@ -426,20 +426,6 @@ def post_process_panoptic_segmentation(self): raise NotImplementedError("Panoptic segmentation post-processing is not implemented for RT-DETR yet.") -class RTDetrImageProcessorKwargs(ImagesKwargs, total=False): - r""" - format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): - Data format of the annotations. One of "coco_detection" or "coco_panoptic". - do_convert_annotations (`bool`, *optional*, defaults to `True`): - Controls whether to convert the annotations to the format expected by the RT_DETR model. Converts the - bounding boxes to the format `(center_x, center_y, width, height)` and in the range `[0, 1]`. - Can be overridden by the `do_convert_annotations` parameter in the `preprocess` method. - """ - - format: str | AnnotationFormat - do_convert_annotations: bool - - @requires(backends=("torch",)) class RTDetrImageProcessorPil(DetrImageProcessorPil): resample = PILImageResampling.BILINEAR diff --git a/src/transformers/models/sam3_lite_text/configuration_sam3_lite_text.py b/src/transformers/models/sam3_lite_text/configuration_sam3_lite_text.py index 696751075611..f77fa99677f4 100644 --- a/src/transformers/models/sam3_lite_text/configuration_sam3_lite_text.py +++ b/src/transformers/models/sam3_lite_text/configuration_sam3_lite_text.py @@ -25,98 +25,7 @@ from ..auto import CONFIG_MAPPING, AutoConfig -@auto_docstring(checkpoint="facebook/sam3_lite_text") -@strict -class Sam3LiteTextViTConfig(PreTrainedConfig): - r""" - rope_theta (`float`, *optional*, defaults to 10000.0): - Base frequency for RoPE. - window_size (`int`, *optional*, defaults to 24): - Window size for windowed attention. - global_attn_indexes (`list[int]`, *optional*, defaults to `[7, 15, 23, 31]`): - Indexes of layers with global attention. - pretrain_image_size (`int`, *optional*, defaults to 336): - Pretrained model image size for position embedding initialization. - hidden_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for hidden states. - """ - - base_config_key = "backbone_config" - model_type = "sam3_vit_model" - - hidden_size: int = 1024 - intermediate_size: int = 4736 - num_hidden_layers: int = 32 - num_attention_heads: int = 16 - num_channels: int = 3 - image_size: int | list[int] | tuple[int, int] = 1008 - patch_size: int | list[int] | tuple[int, int] = 14 - hidden_act: str = "gelu" - layer_norm_eps: float = 1e-6 - attention_dropout: float | int = 0.0 - rope_theta: float = 10000.0 - window_size: int = 24 - global_attn_indexes: list[int] | None = None - layer_scale_init_value: float | None = None - pretrain_image_size: int | list[int] | tuple[int, int] = 336 - hidden_dropout: float | int = 0.0 - initializer_range: float = 0.02 - - def __post_init__(self, **kwargs): - super().__post_init__(**kwargs) - if self.global_attn_indexes is None: - self.global_attn_indexes = [7, 15, 23, 31] - - -@auto_docstring(checkpoint="facebook/sam3_lite_text") -@strict -class Sam3LiteTextVisionConfig(PreTrainedConfig): - r""" - fpn_hidden_size (`int`, *optional*, defaults to 256): - The hidden dimension of the FPN. - backbone_feature_sizes (`List[List[int]]`, *optional*, defaults to `[[288, 288], [144, 144], [72, 72]]`): - The spatial sizes (height, width) of the feature maps from the backbone at different scales. - scale_factors (`list[float]`, *optional*, defaults to `[4.0, 2.0, 1.0, 0.5]`): - Scale factors for FPN multi-scale features. List of scaling factors for each FPN level. - """ - - base_config_key = "vision_config" - model_type = "sam3_vision_model" - sub_configs = {"backbone_config": AutoConfig} - - backbone_config: dict | PreTrainedConfig | None = None - fpn_hidden_size: int = 256 - backbone_feature_sizes: list | None = None - scale_factors: list[float] | None = None - hidden_act: str = "gelu" - layer_norm_eps: float = 1e-6 - initializer_range: float = 0.02 - - def __post_init__(self, **kwargs): - self.scale_factors = [4.0, 2.0, 1.0, 0.5] if self.scale_factors is None else self.scale_factors - if self.backbone_feature_sizes is None: - self.backbone_feature_sizes = [[288, 288], [144, 144], [72, 72]] - - if isinstance(self.backbone_config, dict): - self.backbone_config["model_type"] = self.backbone_config.get("model_type", "sam3_vit_model") - self.backbone_config = CONFIG_MAPPING[self.backbone_config["model_type"]](**self.backbone_config) - elif self.backbone_config is None: - self.backbone_config = CONFIG_MAPPING["sam3_vit_model"]() - - super().__post_init__(**kwargs) - - @property - def image_size(self): - """Image size for the vision encoder.""" - return self.backbone_config.image_size - - @image_size.setter - def image_size(self, value): - """Set the image size and propagate to backbone.""" - self.backbone_config.image_size = value - - -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextGeometryEncoderConfig(PreTrainedConfig): r""" @@ -138,7 +47,7 @@ class Sam3LiteTextGeometryEncoderConfig(PreTrainedConfig): initializer_range: float = 0.02 -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextDETREncoderConfig(PreTrainedConfig): r""" @@ -159,7 +68,7 @@ class Sam3LiteTextDETREncoderConfig(PreTrainedConfig): initializer_range: float = 0.02 -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextDETRDecoderConfig(PreTrainedConfig): r""" @@ -181,7 +90,7 @@ class Sam3LiteTextDETRDecoderConfig(PreTrainedConfig): initializer_range: float = 0.02 -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextMaskDecoderConfig(PreTrainedConfig): r""" @@ -229,7 +138,7 @@ class Sam3LiteTextTextConfig(PreTrainedConfig): repmixer_kernel_size: int = 11 -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextConfig(PreTrainedConfig): r""" diff --git a/src/transformers/models/sam3_lite_text/modeling_sam3_lite_text.py b/src/transformers/models/sam3_lite_text/modeling_sam3_lite_text.py index 5a7b02880edd..05a28e4bea2a 100644 --- a/src/transformers/models/sam3_lite_text/modeling_sam3_lite_text.py +++ b/src/transformers/models/sam3_lite_text/modeling_sam3_lite_text.py @@ -19,7 +19,7 @@ # limitations under the License. import math -from collections.abc import Callable, Iterable +from collections.abc import Callable from dataclasses import dataclass import numpy as np @@ -47,7 +47,6 @@ Sam3LiteTextGeometryEncoderConfig, Sam3LiteTextMaskDecoderConfig, Sam3LiteTextTextConfig, - Sam3LiteTextViTConfig, ) @@ -341,140 +340,6 @@ def forward(self, input_ids: torch.LongTensor) -> torch.Tensor: return hidden_states -class Sam3LiteTextViTRotaryEmbedding(nn.Module): - """ - Vision Rotary Position Embedding for SAM3_LITE_TEXT, following transformers library standards. - Supports 2D (axial) rotary embeddings for spatial dimensions. - """ - - def __init__(self, config: Sam3LiteTextViTConfig, end_x: int, end_y: int, scale: float = 1.0): - super().__init__() - dim = config.hidden_size // config.num_attention_heads - # Ensure even dimension for proper axial splitting - if dim % 4 != 0: - raise ValueError("Dimension must be divisible by 4 for axial RoPE") - self.end_x, self.end_y = end_x, end_y - self.dim = dim - self.rope_theta = config.rope_theta - self.scale = scale - freqs = 1.0 / (config.rope_theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) - - flattened_indices = torch.arange(end_x * end_y, dtype=torch.long) - x_positions = (flattened_indices % end_x) * scale - y_positions = torch.div(flattened_indices, end_x, rounding_mode="floor") * scale - freqs_x = torch.outer(x_positions, freqs).float() - freqs_y = torch.outer(y_positions, freqs).float() - inv_freq = torch.cat([freqs_x, freqs_y], dim=-1) - inv_freq = inv_freq.repeat_interleave(2, dim=-1) - # directly register the cos and sin embeddings as we have a fixed feature shape - self.register_buffer("rope_embeddings_cos", inv_freq.cos(), persistent=False) - self.register_buffer("rope_embeddings_sin", inv_freq.sin(), persistent=False) - - @torch.no_grad() - def forward(self) -> tuple[torch.Tensor, torch.Tensor]: - # As the feature map size is fixed for each stage, we can just return the pre-computed embeddings. - return self.rope_embeddings_cos, self.rope_embeddings_sin - - -class Sam3LiteTextViTPatchEmbeddings(nn.Module): - """ - This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial - `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a - Transformer. - """ - - def __init__(self, config: Sam3LiteTextViTConfig): - super().__init__() - image_size, patch_size = config.pretrain_image_size, config.patch_size - num_channels, hidden_size = config.num_channels, config.hidden_size - - image_size = image_size if isinstance(image_size, Iterable) else (image_size, image_size) - patch_size = patch_size if isinstance(patch_size, Iterable) else (patch_size, patch_size) - num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) - self.image_size = image_size - self.patch_size = patch_size - self.num_channels = num_channels - self.num_patches = num_patches - - self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size, bias=False) - - def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: - embeddings = self.projection(pixel_values.to(self.projection.weight.dtype)).flatten(2).transpose(1, 2) - return embeddings - - -class Sam3LiteTextViTEmbeddings(nn.Module): - """ - Construct the patch embeddings and position embeddings for SAM3_LITE_TEXT ViT. - - Position embeddings are tiled (not interpolated) when resizing to match different input sizes. - """ - - def __init__(self, config: Sam3LiteTextViTConfig): - super().__init__() - - self.patch_embeddings = Sam3LiteTextViTPatchEmbeddings(config) - num_patches = self.patch_embeddings.num_patches - self.position_embeddings = nn.Parameter( - torch.randn(1, num_patches, config.hidden_size) - ) # !Remove cls token in convert weights! - - self.dropout = nn.Dropout(config.hidden_dropout) - self.patch_size = config.patch_size - - def _tile_position_embeddings( - self, - position_embeddings: torch.Tensor, - height: int, - width: int, - ) -> torch.Tensor: - """ - Tile position embeddings to match target spatial dimensions. - Args: - position_embeddings: Shape [1, num_pretrain_patches, hidden_size] - height: Target height in patches - width: Target width in patches - - Returns: - Shape [1, height * width, hidden_size] - """ - pretrain_size = int(position_embeddings.shape[1] ** 0.5) - - # Skip tiling if sizes match (but always tile during tracing for consistent graph) - if not torch.jit.is_tracing() and pretrain_size == height and pretrain_size == width: - return position_embeddings.reshape(1, height * width, -1) - - # Tile position embeddings to match target spatial dimensions - hidden_size = position_embeddings.shape[-1] - pos_embed = position_embeddings.reshape(1, pretrain_size, pretrain_size, hidden_size).permute(0, 3, 1, 2) - repeat_h = height // pretrain_size + 1 - repeat_w = width // pretrain_size + 1 - pos_embed = pos_embed.tile([1, 1, repeat_h, repeat_w])[:, :, :height, :width] - return pos_embed.permute(0, 2, 3, 1).reshape(1, height * width, hidden_size) - - def forward( - self, - pixel_values: torch.Tensor, - interpolate_pos_encoding: bool = False, - ) -> torch.Tensor: - height, width = pixel_values.shape[-2:] - embeddings = self.patch_embeddings(pixel_values) - - # Calculate spatial dimensions in patches - height_patches = height // self.patch_size - width_patches = width // self.patch_size - - position_embeddings = self._tile_position_embeddings( - self.position_embeddings, - height_patches, - width_patches, - ) - embeddings = embeddings + position_embeddings - embeddings = self.dropout(embeddings) - - return embeddings - - @auto_docstring class Sam3LiteTextPreTrainedModel(PreTrainedModel): config_class = Sam3LiteTextConfig @@ -490,21 +355,6 @@ class Sam3LiteTextPreTrainedModel(PreTrainedModel): @torch.no_grad() def _init_weights(self, module): super()._init_weights(module) - if isinstance(module, Sam3LiteTextViTEmbeddings): - init.normal_(module.position_embeddings, mean=0.0, std=self.config.initializer_range) - elif isinstance(module, Sam3LiteTextViTRotaryEmbedding): - end_x, end_y = module.end_x, module.end_y - dim = module.dim - freqs = 1.0 / (module.rope_theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) - flattened_indices = torch.arange(end_x * end_y, dtype=torch.long) - x_positions = (flattened_indices % end_x) * module.scale - y_positions = torch.div(flattened_indices, end_x, rounding_mode="floor") * module.scale - freqs_x = torch.outer(x_positions, freqs).float() - freqs_y = torch.outer(y_positions, freqs).float() - inv_freq = torch.cat([freqs_x, freqs_y], dim=-1) - inv_freq = inv_freq.repeat_interleave(2, dim=-1) - init.copy_(module.rope_embeddings_cos, inv_freq.cos()) - init.copy_(module.rope_embeddings_sin, inv_freq.sin()) if isinstance(module, Sam3LiteTextTextPositionEmbedding): init.normal_(module.position_embedding, std=module.position_embedding.shape[-1] ** -0.5) elif isinstance(module, Sam3LiteTextTextModel): diff --git a/src/transformers/models/sam3_lite_text/modular_sam3_lite_text.py b/src/transformers/models/sam3_lite_text/modular_sam3_lite_text.py index 46408464a333..4e830a6d5ec3 100644 --- a/src/transformers/models/sam3_lite_text/modular_sam3_lite_text.py +++ b/src/transformers/models/sam3_lite_text/modular_sam3_lite_text.py @@ -24,6 +24,7 @@ from ...configuration_utils import PreTrainedConfig from ...masking_utils import create_bidirectional_mask from ...modeling_outputs import BaseModelOutputWithPooling +from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import auto_docstring from ...utils.generic import TransformersKwargs, merge_with_config_defaults @@ -39,116 +40,25 @@ from ..siglip.modeling_siglip import SiglipAttention, SiglipEncoderLayer, SiglipMLP -@auto_docstring(checkpoint="facebook/sam3_lite_text") -@strict -class Sam3LiteTextViTConfig(PreTrainedConfig): - r""" - rope_theta (`float`, *optional*, defaults to 10000.0): - Base frequency for RoPE. - window_size (`int`, *optional*, defaults to 24): - Window size for windowed attention. - global_attn_indexes (`list[int]`, *optional*, defaults to `[7, 15, 23, 31]`): - Indexes of layers with global attention. - pretrain_image_size (`int`, *optional*, defaults to 336): - Pretrained model image size for position embedding initialization. - hidden_dropout (`float`, *optional*, defaults to 0.0): - Dropout probability for hidden states. - """ - - base_config_key = "backbone_config" - model_type = "sam3_vit_model" - - hidden_size: int = 1024 - intermediate_size: int = 4736 - num_hidden_layers: int = 32 - num_attention_heads: int = 16 - num_channels: int = 3 - image_size: int | list[int] | tuple[int, int] = 1008 - patch_size: int | list[int] | tuple[int, int] = 14 - hidden_act: str = "gelu" - layer_norm_eps: float = 1e-6 - attention_dropout: float | int = 0.0 - rope_theta: float = 10000.0 - window_size: int = 24 - global_attn_indexes: list[int] | None = None - layer_scale_init_value: float | None = None - pretrain_image_size: int | list[int] | tuple[int, int] = 336 - hidden_dropout: float | int = 0.0 - initializer_range: float = 0.02 - - def __post_init__(self, **kwargs): - super().__post_init__(**kwargs) - if self.global_attn_indexes is None: - self.global_attn_indexes = [7, 15, 23, 31] - - -@auto_docstring(checkpoint="facebook/sam3_lite_text") -@strict -class Sam3LiteTextVisionConfig(PreTrainedConfig): - r""" - fpn_hidden_size (`int`, *optional*, defaults to 256): - The hidden dimension of the FPN. - backbone_feature_sizes (`List[List[int]]`, *optional*, defaults to `[[288, 288], [144, 144], [72, 72]]`): - The spatial sizes (height, width) of the feature maps from the backbone at different scales. - scale_factors (`list[float]`, *optional*, defaults to `[4.0, 2.0, 1.0, 0.5]`): - Scale factors for FPN multi-scale features. List of scaling factors for each FPN level. - """ - - base_config_key = "vision_config" - model_type = "sam3_vision_model" - sub_configs = {"backbone_config": AutoConfig} - - backbone_config: dict | PreTrainedConfig | None = None - fpn_hidden_size: int = 256 - backbone_feature_sizes: list | None = None - scale_factors: list[float] | None = None - hidden_act: str = "gelu" - layer_norm_eps: float = 1e-6 - initializer_range: float = 0.02 - - def __post_init__(self, **kwargs): - self.scale_factors = [4.0, 2.0, 1.0, 0.5] if self.scale_factors is None else self.scale_factors - if self.backbone_feature_sizes is None: - self.backbone_feature_sizes = [[288, 288], [144, 144], [72, 72]] - - if isinstance(self.backbone_config, dict): - self.backbone_config["model_type"] = self.backbone_config.get("model_type", "sam3_vit_model") - self.backbone_config = CONFIG_MAPPING[self.backbone_config["model_type"]](**self.backbone_config) - elif self.backbone_config is None: - self.backbone_config = CONFIG_MAPPING["sam3_vit_model"]() - - super().__post_init__(**kwargs) - - @property - def image_size(self): - """Image size for the vision encoder.""" - return self.backbone_config.image_size - - @image_size.setter - def image_size(self, value): - """Set the image size and propagate to backbone.""" - self.backbone_config.image_size = value - - -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextGeometryEncoderConfig(Sam3GeometryEncoderConfig): pass -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextDETREncoderConfig(Sam3DETREncoderConfig): pass -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextDETRDecoderConfig(Sam3DETRDecoderConfig): pass -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextMaskDecoderConfig(Sam3MaskDecoderConfig): pass @@ -184,7 +94,7 @@ class Sam3LiteTextTextConfig(PreTrainedConfig): repmixer_kernel_size: int = 11 -@auto_docstring(checkpoint="facebook/sam3_lite_text") +@auto_docstring(checkpoint="yonigozlan/sam3-litetext-s0") @strict class Sam3LiteTextConfig(PreTrainedConfig): r""" @@ -444,7 +354,7 @@ class Sam3LiteTextPreTrainedModel(Sam3PreTrainedModel): @torch.no_grad() def _init_weights(self, module): - super()._init_weights(module) + PreTrainedModel._init_weights(module) if isinstance(module, Sam3LiteTextTextPositionEmbedding): init.normal_(module.position_embedding, std=module.position_embedding.shape[-1] ** -0.5) elif isinstance(module, Sam3LiteTextTextModel): diff --git a/src/transformers/models/segformer/modular_segformer.py b/src/transformers/models/segformer/modular_segformer.py index d7f339ea6e42..414dc58e8c52 100644 --- a/src/transformers/models/segformer/modular_segformer.py +++ b/src/transformers/models/segformer/modular_segformer.py @@ -31,22 +31,10 @@ PILImageResampling, SizeDict, ) -from ...processing_utils import ImagesKwargs from ...utils import TensorType from ...utils.import_utils import requires -class SegformerImageProcessorKwargs(ImagesKwargs, total=False): - r""" - do_reduce_labels (`bool`, *optional*, defaults to `self.do_reduce_labels`): - Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 - is used for background, and background itself is not included in all classes of a dataset (e.g. - ADE20k). The background label will be replaced by 255. - """ - - do_reduce_labels: bool - - class SegformerImageProcessor(BeitImageProcessor): resample = PILImageResampling.BILINEAR image_mean = IMAGENET_DEFAULT_MEAN diff --git a/src/transformers/models/smolvlm/modular_smolvlm.py b/src/transformers/models/smolvlm/modular_smolvlm.py index 9c572cc9d877..cf91863c56a7 100644 --- a/src/transformers/models/smolvlm/modular_smolvlm.py +++ b/src/transformers/models/smolvlm/modular_smolvlm.py @@ -22,7 +22,7 @@ from ...generation import GenerationConfig from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import BaseModelOutputWithPooling -from ...processing_utils import ImagesKwargs, Unpack +from ...processing_utils import Unpack from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging, torch_compilable_check from ..idefics3.configuration_idefics3 import Idefics3Config, Idefics3VisionConfig from ..idefics3.image_processing_idefics3 import Idefics3ImageProcessor @@ -91,22 +91,6 @@ class SmolVLMConfig(Idefics3Config): model_type = "smolvlm" -class SmolVLMImageProcessorKwargs(ImagesKwargs, total=False): - """ - do_image_splitting (`bool`, *optional*, defaults to `True`): - Whether to split the image into sub-images concatenated with the original image. They are split into patches - such that each patch has a size of `max_image_size["height"]` x `max_image_size["width"]`. - max_image_size (`Dict`, *optional*, defaults to `{"longest_edge": 364}`): - Maximum resolution of the patches of images accepted by the model. This is a dictionary containing the key "longest_edge". - return_row_col_info (`bool`, *optional*, defaults to `False`): - Whether to return the row and column information of the images. - """ - - do_image_splitting: bool - max_image_size: dict[str, int] - return_row_col_info: bool - - class SmolVLMImageProcessor(Idefics3ImageProcessor): pass diff --git a/src/transformers/models/solar_open/modeling_solar_open.py b/src/transformers/models/solar_open/modeling_solar_open.py index dfa30292455f..0eb50021ecd6 100644 --- a/src/transformers/models/solar_open/modeling_solar_open.py +++ b/src/transformers/models/solar_open/modeling_solar_open.py @@ -200,7 +200,7 @@ def route_tokens_to_experts(self, router_logits): .expand(-1, self.n_group, self.n_routed_experts // self.n_group) .reshape(-1, self.n_routed_experts) ) - scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), 0.0) + scores_for_choice = router_logits_for_choice.masked_fill(~score_mask.bool(), float("-inf")) topk_indices = torch.topk(scores_for_choice, k=self.top_k, dim=-1, sorted=False)[1] topk_weights = router_logits.gather(1, topk_indices) if self.norm_topk_prob: diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index 2e0dddc17876..90ab31a30665 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -706,7 +706,7 @@ def _init_weights(self, module): init.copy_(getattr(module, f"{layer_type}_inv_freq"), curr_inv_freq) init.copy_(getattr(module, f"{layer_type}_original_inv_freq"), curr_inv_freq) - def prepare_decoder_input_ids_from_labels(self, input_ids): + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): """ Shifts input_ids to the right, prepends the decoder_start_token_id, and handles pad_token_id replacement for labels that were -100. @@ -720,8 +720,8 @@ def prepare_decoder_input_ids_from_labels(self, input_ids): raise ValueError("self.model.config.decoder.bos_token_id has to be defined. ") # shift inputs to the right - shifted_input_ids = input_ids.new_zeros(input_ids.shape) - shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() + shifted_input_ids = labels.new_zeros(labels.shape) + shifted_input_ids[..., 1:] = labels[..., :-1].clone() shifted_input_ids[..., 0] = decoder_start_token_id if pad_token_id is None: diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 2f0f3720a7cd..53cf7518901a 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -513,7 +513,7 @@ def _init_weights(self, module): init.copy_(getattr(module, f"{layer_type}_inv_freq"), curr_inv_freq) init.copy_(getattr(module, f"{layer_type}_original_inv_freq"), curr_inv_freq) - def prepare_decoder_input_ids_from_labels(self, input_ids): + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): """ Shifts input_ids to the right, prepends the decoder_start_token_id, and handles pad_token_id replacement for labels that were -100. @@ -527,8 +527,8 @@ def prepare_decoder_input_ids_from_labels(self, input_ids): raise ValueError("self.model.config.decoder.bos_token_id has to be defined. ") # shift inputs to the right - shifted_input_ids = input_ids.new_zeros(input_ids.shape) - shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() + shifted_input_ids = labels.new_zeros(labels.shape) + shifted_input_ids[..., 1:] = labels[..., :-1].clone() shifted_input_ids[..., 0] = decoder_start_token_id if pad_token_id is None: diff --git a/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py b/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py index 9990852d83cf..859dc58e5873 100644 --- a/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py +++ b/src/transformers/models/vibevoice_acoustic_tokenizer/feature_extraction_vibevoice_acoustic_tokenizer.py @@ -73,6 +73,7 @@ def __call__( max_length: int | None = None, return_attention_mask: bool | None = True, return_tensors: str | None = "pt", + **kwargs, ) -> BatchFeature: """ Args: diff --git a/src/transformers/models/vibevoice_asr/modeling_vibevoice_asr.py b/src/transformers/models/vibevoice_asr/modeling_vibevoice_asr.py index 703bb6ca5130..3d26d0fbe9f3 100644 --- a/src/transformers/models/vibevoice_asr/modeling_vibevoice_asr.py +++ b/src/transformers/models/vibevoice_asr/modeling_vibevoice_asr.py @@ -256,6 +256,7 @@ def _init_weights(self, module): ) class VibeVoiceAsrForConditionalGeneration(VibeVoiceAsrPreTrainedModel, GenerationMixin): _keep_in_fp32_modules_strict = None + _supports_attention_backend = True _tp_plan = None _pp_plan = None diff --git a/src/transformers/models/vibevoice_asr/modular_vibevoice_asr.py b/src/transformers/models/vibevoice_asr/modular_vibevoice_asr.py index fc9c960c1033..5fb92a1d4f1b 100644 --- a/src/transformers/models/vibevoice_asr/modular_vibevoice_asr.py +++ b/src/transformers/models/vibevoice_asr/modular_vibevoice_asr.py @@ -167,6 +167,8 @@ class VibeVoiceAsrPreTrainedModel(VibeVoiceAcousticTokenizerPreTrainedModel): """ ) class VibeVoiceAsrForConditionalGeneration(AudioFlamingo3ForConditionalGeneration): + _supports_attention_backend = True + def __init__(self, config: VibeVoiceAsrConfig): super().__init__(config) self.acoustic_tokenizer_encoder = AutoModel.from_config(config.acoustic_tokenizer_encoder_config) diff --git a/src/transformers/models/video_llama_3/modular_video_llama_3.py b/src/transformers/models/video_llama_3/modular_video_llama_3.py index 4eef74580c87..c4a9e40bc8f0 100644 --- a/src/transformers/models/video_llama_3/modular_video_llama_3.py +++ b/src/transformers/models/video_llama_3/modular_video_llama_3.py @@ -37,7 +37,7 @@ ) from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ModelOutput from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel -from ...processing_utils import ImagesKwargs, Unpack +from ...processing_utils import Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import ( TensorType, @@ -55,7 +55,7 @@ from ..auto import CONFIG_MAPPING, AutoConfig from ..auto.modeling_auto import AutoModel from ..qwen2_vl.image_processing_pil_qwen2_vl import Qwen2VLImageProcessorPil -from ..qwen2_vl.image_processing_qwen2_vl import Qwen2VLImageProcessor, smart_resize +from ..qwen2_vl.image_processing_qwen2_vl import Qwen2VLImageProcessor, Qwen2VLImageProcessorKwargs, smart_resize from ..qwen2_vl.modeling_qwen2_vl import ( Qwen2VLForConditionalGeneration, Qwen2VLModel, @@ -1107,25 +1107,8 @@ def model_input_names(self): raise AttributeError("VideoLlama doesn't need to override it") -class VideoLlama3ImageProcessorKwargs(ImagesKwargs, total=False): - r""" - min_pixels (`int`, *optional*, defaults to `56 * 56`): - The min pixels of the image to resize the image. - max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`): - The max pixels of the image to resize the image. - patch_size (`int`, *optional*, defaults to 14): - The spatial patch size of the vision encoder. - temporal_patch_size (`int`, *optional*, defaults to 2): - The temporal patch size of the vision encoder. - merge_size (`int`, *optional*, defaults to 2): - The merge size of the vision encoder to llm encoder. - """ - - min_pixels: int - max_pixels: int - patch_size: int - temporal_patch_size: int - merge_size: int +class VideoLlama3ImageProcessorKwargs(Qwen2VLImageProcessorKwargs): + pass class VideoLlama3ImageProcessorPil(Qwen2VLImageProcessorPil): diff --git a/src/transformers/models/yolos/image_processing_pil_yolos.py b/src/transformers/models/yolos/image_processing_pil_yolos.py index 219348363ea3..f42fb5a63701 100644 --- a/src/transformers/models/yolos/image_processing_pil_yolos.py +++ b/src/transformers/models/yolos/image_processing_pil_yolos.py @@ -44,8 +44,9 @@ import torch from torch import nn +SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) + -# Adapted from transformers.models.yolos.image_processing_yolos.YolosImageProcessorKwargs class YolosImageProcessorKwargs(ImagesKwargs, total=False): r""" format (`str`, *optional*, defaults to `AnnotationFormat.COCO_DETECTION`): @@ -60,9 +61,6 @@ class YolosImageProcessorKwargs(ImagesKwargs, total=False): do_convert_annotations: bool -SUPPORTED_ANNOTATION_FORMATS = (AnnotationFormat.COCO_DETECTION, AnnotationFormat.COCO_PANOPTIC) - - # inspired by https://github.com/facebookresearch/yolos/blob/master/datasets/coco.py#L33 def convert_coco_poly_to_mask(segmentations, height: int, width: int) -> np.ndarray: """ diff --git a/src/transformers/pipelines/text_to_audio.py b/src/transformers/pipelines/text_to_audio.py index d768126be33b..a4d70912dd63 100644 --- a/src/transformers/pipelines/text_to_audio.py +++ b/src/transformers/pipelines/text_to_audio.py @@ -179,6 +179,7 @@ def preprocess(self, text, **kwargs): # Add speaker ID if needed and user didn't insert at start of text if self.model.config.model_type == "csm": text = [f"[0]{t}" if not t.startswith("[") else t for t in text] + kwargs.setdefault("add_special_tokens", True) if self.model.config.model_type == "dia": text = [f"[S1] {t}" if not t.startswith("[") else t for t in text] output = preprocessor(text, **kwargs, return_tensors="pt") diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index f7980a2bce25..25619ca55b3f 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -63,7 +63,6 @@ ) from .utils.chat_parsing_utils import recursive_parse from .utils.chat_template_utils import render_jinja_template -from .utils.import_utils import PROTOBUF_IMPORT_ERROR if TYPE_CHECKING: @@ -76,8 +75,7 @@ def import_protobuf_decode_error(error_message=""): from google.protobuf.message import DecodeError return DecodeError - else: - raise ImportError(PROTOBUF_IMPORT_ERROR.format(error_message)) + return () def flatten(arr: list): diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 8665b27acfa9..f434d78d4040 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -712,6 +712,8 @@ def _build_accelerator_args(self, **kwargs) -> dict[str, Any]: ddp_kwargs["bucket_cap_mb"] = self.args.ddp_bucket_cap_mb if self.args.ddp_broadcast_buffers is not None: ddp_kwargs["broadcast_buffers"] = self.args.ddp_broadcast_buffers + if self.args.ddp_static_graph is not None: + ddp_kwargs["static_graph"] = self.args.ddp_static_graph args["kwargs_handlers"] = [DistributedDataParallelKwargs(**ddp_kwargs)] diff --git a/src/transformers/trainer_callback.py b/src/transformers/trainer_callback.py index 6c4f72859264..598f24a604c9 100644 --- a/src/transformers/trainer_callback.py +++ b/src/transformers/trainer_callback.py @@ -681,7 +681,7 @@ def on_log(self, args, state, control, logs=None, **kwargs): f"[String too long to display, length: {len(v)} > {self.max_str_len}. " "Consider increasing `max_str_len` if needed.]" ) - if isinstance(v, float): + elif isinstance(v, float): # Format floats for better readability shallow_logs[k] = f"{v:.4g}" else: diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index e66af264b89a..1a5924c723ab 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -637,6 +637,9 @@ class TrainingArguments: ddp_broadcast_buffers (`bool`, *optional*): When using distributed training, the value of the flag `broadcast_buffers` passed to `DistributedDataParallel`. Will default to `False` if gradient checkpointing is used, `True` otherwise. + ddp_static_graph (`bool`, *optional*): + When using distributed training, the value of the flag `static_graph` passed to + `DistributedDataParallel`. ddp_backend (`str`, *optional*): The backend to use for distributed training. Must be one of `"nccl"`, `"mpi"`, `"xccl"`, `"gloo"`, `"hccl"`. ddp_timeout (`int`, *optional*, defaults to 1800): @@ -1376,6 +1379,15 @@ class TrainingArguments: ) }, ) + ddp_static_graph: bool | None = field( + default=None, + metadata={ + "help": ( + "When using distributed training, the value of the flag `static_graph` passed to " + "`DistributedDataParallel`." + ) + }, + ) ddp_backend: str | None = field( default=None, metadata={ diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 908fb69fa2f8..bf085d87498c 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -1848,12 +1848,16 @@ class FourOverSixConfig(QuantizationConfigMixin): error. Refer to the original publication for more details: https://arxiv.org/abs/2512.02010. Args: + activation_dtype (`str`, *optional*): + Data type to use when quantizing activation tensors. If not provided, `dtype` is used. activation_scale_rule (`str`, *optional*): Scaling rule to use when selecting a scale for blocks in activation tensors. If not provided, `scale_rule` is used. dtype (`str`, default "nvfp4", *optional*, defaults to `"nvfp4"`): The data type to use for the layer's weights, activations, and tensors. Can be `"nvfp4"` or `"mxfp4"`. + gradient_dtype (`str`, *optional*): + Data type to use when quantizing gradient tensors. If not provided, `dtype` is used. gradient_scale_rule (`str`, *optional*): Scaling rule to use when selecting a scale for blocks in gradient tensors. If not provided, `scale_rule` is used. @@ -1876,6 +1880,8 @@ class FourOverSixConfig(QuantizationConfigMixin): Rule to use when selecting block scales. Can be `"mse"`, `"mae"`, or `"abs_max"` for Four Over Six, `"static_6"` for default NVFP4 quantization, or `"static_4"` to scale all blocks to a maximum value of 4. + weight_dtype (`str`, *optional*): + Data type to use when quantizing weight tensors. If not provided, `dtype` is used. weight_scale_2d (`bool`, default False, *optional*, defaults to `False`): Whether to compute scale factors on weight tensors in 2D blocks. This should be done during training. @@ -1892,14 +1898,17 @@ class FourOverSixConfig(QuantizationConfigMixin): def __init__( self, + activation_dtype: str | None = None, activation_scale_rule: str | None = None, dtype: str = "nvfp4", + gradient_dtype: str | None = None, gradient_scale_rule: str | None = None, keep_master_weights: bool = False, matmul_backend: str | None = None, output_dtype: str | None = "bfloat16", quantize_backend: str | None = None, scale_rule: str = "mse", + weight_dtype: str | None = None, weight_scale_2d: bool = False, weight_scale_rule: str | None = None, module_config_overrides: dict[str, dict[str, Any]] | None = None, @@ -1908,14 +1917,17 @@ def __init__( ): self.quant_method = QuantizationMethod.FOUR_OVER_SIX + self.activation_dtype = activation_dtype self.activation_scale_rule = activation_scale_rule self.dtype = dtype + self.gradient_dtype = gradient_dtype self.gradient_scale_rule = gradient_scale_rule self.keep_master_weights = keep_master_weights self.matmul_backend = matmul_backend self.quantize_backend = quantize_backend self.output_dtype = output_dtype self.scale_rule = scale_rule + self.weight_dtype = weight_dtype self.weight_scale_2d = weight_scale_2d self.weight_scale_rule = weight_scale_rule self.module_config_overrides = module_config_overrides diff --git a/tests/cli/test_serve.py b/tests/cli/test_serve.py index c7f9b96f4790..c54de16b32bd 100644 --- a/tests/cli/test_serve.py +++ b/tests/cli/test_serve.py @@ -32,11 +32,12 @@ from transformers.cli.serving.server import build_server from transformers.cli.serving.transcription import TranscriptionHandler from transformers.cli.serving.utils import ( + _TOOL_CALL_FALLBACKS, BaseHandler, GenerationState, Modality, - ToolCallParser, - detect_tool_format, + get_tool_call_config, + parse_tool_calls, ) from transformers.testing_utils import ( require_librosa, @@ -47,6 +48,7 @@ require_vision, slow, ) +from transformers.utils.chat_parsing_utils import recursive_parse from transformers.utils.import_utils import is_serve_available @@ -427,6 +429,36 @@ def test_unsupported_fields_warns(self): self.assertTrue(any("audio" in msg for msg in cm.output)) +class TestResolveModel(unittest.TestCase): + def _make_handler(self, force_model=None): + mm = MagicMock() + mm.force_model = force_model + mm.process_model_name.side_effect = ModelManager.process_model_name + mm.load_model_and_processor.return_value = (MagicMock(), MagicMock()) + return ChatCompletionHandler(model_manager=mm, generation_state=GenerationState()) + + def test_force_model_overrides_when_model_omitted(self): + handler = self._make_handler(force_model="org/pinned") + body = {} + model_id, _, _ = handler._resolve_model(body) + self.assertEqual(model_id, "org/pinned@main") + self.assertEqual(body["model"], "org/pinned") + + def test_force_model_allows_matching_request(self): + handler = self._make_handler(force_model="org/pinned") + body = {"model": "org/pinned"} + model_id, _, _ = handler._resolve_model(body) + self.assertEqual(model_id, "org/pinned@main") + + def test_force_model_rejects_mismatched_request(self): + handler = self._make_handler(force_model="org/pinned") + with self.assertRaises(HTTPException) as ctx: + handler._resolve_model({"model": "other/model"}) + self.assertEqual(ctx.exception.status_code, 400) + self.assertIn("org/pinned", ctx.exception.detail) + self.assertIn("other/model", ctx.exception.detail) + + class TestModelManager(unittest.TestCase): def test_process_model_name_adds_main(self): self.assertEqual(ModelManager.process_model_name("org/model"), "org/model@main") @@ -504,130 +536,7 @@ def test_chunk_to_sse_wraps_plain_string(self): self.assertEqual(result, "data: hello\n\n") -QWEN_TOOL_FORMAT = {"start": "", "end": ""} - - @require_serve -class TestToolParser(unittest.TestCase): - def test_detect_tool_format_qwen(self): - model = MagicMock() - model.config.architectures = ["Qwen2ForCausalLM"] - fmt = detect_tool_format(model) - self.assertEqual(fmt, QWEN_TOOL_FORMAT) - - def test_detect_tool_format_unsupported(self): - model = MagicMock() - model.config.architectures = ["LlamaForCausalLM"] - self.assertIsNone(detect_tool_format(model)) - - def test_parser_start_token(self): - parser = ToolCallParser(QWEN_TOOL_FORMAT) - result = parser.feed("") - self.assertIs(result, ToolCallParser.CONSUMED) - - def test_parser_end_token(self): - parser = ToolCallParser(QWEN_TOOL_FORMAT) - parser.feed("") - result = parser.feed("") - self.assertIs(result, ToolCallParser.CONSUMED) - - def test_parser_buffers_until_end(self): - parser = ToolCallParser(QWEN_TOOL_FORMAT) - parser.feed("") - # Intermediate tokens are buffered - result = parser.feed('{"name": "my_tool", "arguments": {"x": 1}}') - self.assertIs(result, ToolCallParser.CONSUMED) - # Tool call is emitted on end token - result = parser.feed("") - self.assertIsNot(result, ToolCallParser.CONSUMED) - self.assertEqual(result["name"], "my_tool") - - def test_parser_normal_text_returns_none(self): - parser = ToolCallParser(QWEN_TOOL_FORMAT) - result = parser.feed("Hello world") - self.assertIsNone(result) - - def test_parser_full_flow(self): - """Simulate a complete tool call token sequence.""" - - parser = ToolCallParser(QWEN_TOOL_FORMAT) - tool_calls = [] - - for token in [ - "", - '{"name": "get_weather",', - ' "arguments": {', - '"city": "Paris"', - "}}", - "\n", - "", - ]: - result = parser.feed(token) - if result is not None and result is not ToolCallParser.CONSUMED: - tool_calls.append(result) - - # Single tool call emitted on with both name and arguments - self.assertEqual(len(tool_calls), 1) - self.assertEqual(tool_calls[0]["name"], "get_weather") - self.assertIn("Paris", tool_calls[0]["arguments"]) - - def test_parse_tool_calls_from_text(self): - """Non-streaming tool call parsing from complete text.""" - - text = '\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n' - calls = ToolCallParser.parse(text, QWEN_TOOL_FORMAT) - self.assertIsNotNone(calls) - self.assertEqual(len(calls), 1) - self.assertEqual(calls[0]["name"], "get_weather") - self.assertIn("Paris", calls[0]["arguments"]) - - def test_parse_tool_calls_no_tool_call(self): - """Non-streaming: normal text returns None.""" - - calls = ToolCallParser.parse("Hello, how can I help?", QWEN_TOOL_FORMAT) - self.assertIsNone(calls) - - def test_parse_multiple_tool_calls(self): - """Non-streaming: multiple tool calls in one response.""" - - text = ( - '\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n\n' - '\n{"name": "get_weather", "arguments": {"city": "London"}}\n' - ) - calls = ToolCallParser.parse(text, QWEN_TOOL_FORMAT) - self.assertIsNotNone(calls) - self.assertEqual(len(calls), 2) - self.assertEqual(calls[0]["name"], "get_weather") - self.assertIn("Paris", calls[0]["arguments"]) - self.assertEqual(calls[1]["name"], "get_weather") - self.assertIn("London", calls[1]["arguments"]) - - def test_feed_multiple_tool_calls(self): - """Streaming: multiple tool calls emitted sequentially.""" - - parser = ToolCallParser(QWEN_TOOL_FORMAT) - tool_calls = [] - - tokens = [ - "", - '{"name": "get_weather", "arguments": {"city": "Paris"}}', - "", - "", - '{"name": "get_weather", "arguments": {"city": "London"}}', - "", - ] - for token in tokens: - result = parser.feed(token) - if result is not None and result is not ToolCallParser.CONSUMED: - tool_calls.append(result) - - self.assertEqual(len(tool_calls), 2) - self.assertEqual(tool_calls[0]["name"], "get_weather") - self.assertIn("Paris", tool_calls[0]["arguments"]) - self.assertEqual(tool_calls[1]["name"], "get_weather") - self.assertIn("London", tool_calls[1]["arguments"]) - - @require_serve class TestAppRoutes(unittest.TestCase): @classmethod @@ -784,110 +693,6 @@ def test_streaming_usage(self): self.assertGreater(last.usage.completion_tokens, 0) self.assertEqual(last.usage.total_tokens, last.usage.prompt_tokens + last.usage.completion_tokens) - def test_tool_call(self): - """Tool calls should be parsed and emitted as ChoiceDeltaToolCall objects.""" - # Qwen2.5-0.5B-Instruct supports tools (Qwen family) - tool_def = { - "function": { - "name": "get_weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - "description": "Get the weather for a city.", - }, - "type": "function", - } - chunks = list( - self.client.chat.completions.create( - model=self.MODEL, - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - stream=True, - max_tokens=50, - temperature=0.0, - tools=[tool_def], - ) - ) - - # First chunk should have role="assistant" - self.assertEqual(chunks[0].choices[0].delta.role, "assistant") - - # Model should make a tool call for this prompt - tool_chunks = [c for c in chunks if c.choices[0].delta.tool_calls] - self.assertGreater(len(tool_chunks), 0, "Model did not produce a tool call") - - # First tool call delta should have the function name - first_tool = tool_chunks[0].choices[0].delta.tool_calls[0] - self.assertEqual(first_tool.function.name, "get_weather") - - # finish_reason should be "tool_calls" - last = chunks[-1] - self.assertEqual(last.choices[0].finish_reason, "tool_calls") - - # Arguments should be valid JSON with no trailing brace - args_json = first_tool.function.arguments - import json as json_mod - - parsed_args = json_mod.loads(args_json) - self.assertIsInstance(parsed_args, dict) - - def test_tool_call_non_streaming(self): - """Non-streaming tool calls should return tool_calls in the message.""" - tool_def = { - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - "description": "Get the weather for a city.", - }, - "type": "function", - } - resp = self.client.chat.completions.create( - model=self.MODEL, - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - stream=False, - max_tokens=50, - temperature=0.0, - tools=[tool_def], - ) - self.assertEqual(resp.choices[0].finish_reason, "tool_calls") - self.assertIsNotNone(resp.choices[0].message.tool_calls) - tc = resp.choices[0].message.tool_calls[0] - self.assertEqual(tc.function.name, "get_weather") - - import json as json_mod - - parsed_args = json_mod.loads(tc.function.arguments) - self.assertIsInstance(parsed_args, dict) - - def test_tool_call_multi(self): - """Model should be able to call multiple tools when asked.""" - tool_def = { - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - "description": "Get the weather for a city.", - }, - "type": "function", - } - # Ask for two cities to encourage multiple tool calls - chunks = list( - self.client.chat.completions.create( - model=self.MODEL, - messages=[{"role": "user", "content": "What is the weather in Paris and London?"}], - stream=True, - max_tokens=100, - temperature=0.0, - tools=[tool_def], - ) - ) - tool_chunks = [c for c in chunks if c.choices[0].delta.tool_calls] - # Should have two tool calls — one for Paris, one for London - self.assertEqual(len(tool_chunks), 2, f"Expected 2 tool calls, got {len(tool_chunks)}") - cities = {tc.choices[0].delta.tool_calls[0].function.name for tc in tool_chunks} - self.assertEqual(cities, {"get_weather"}) - last = chunks[-1] - self.assertEqual(last.choices[0].finish_reason, "tool_calls") - def test_concurrent_non_streaming(self): """Two concurrent non-streaming requests should both complete without interference.""" import concurrent.futures @@ -978,19 +783,19 @@ def _make_handler(self): def test_string_input(self): handler = self._make_handler() - msgs = handler._input_to_messages({"input": "Hello"}) + msgs = handler._normalize_input({"input": "Hello"}) self.assertEqual(msgs, [{"role": "user", "content": "Hello"}]) def test_string_input_with_instructions(self): handler = self._make_handler() - msgs = handler._input_to_messages({"input": "Hello", "instructions": "Be brief"}) + msgs = handler._normalize_input({"input": "Hello", "instructions": "Be brief"}) self.assertEqual(len(msgs), 2) self.assertEqual(msgs[0], {"role": "system", "content": "Be brief"}) self.assertEqual(msgs[1], {"role": "user", "content": "Hello"}) def test_list_input(self): handler = self._make_handler() - msgs = handler._input_to_messages( + msgs = handler._normalize_input( {"input": [{"role": "user", "content": "A"}, {"role": "assistant", "content": "B"}]} ) self.assertEqual(len(msgs), 2) @@ -998,14 +803,14 @@ def test_list_input(self): def test_list_input_with_instructions_prepends_system(self): handler = self._make_handler() - msgs = handler._input_to_messages({"input": [{"role": "user", "content": "Hi"}], "instructions": "Be helpful"}) + msgs = handler._normalize_input({"input": [{"role": "user", "content": "Hi"}], "instructions": "Be helpful"}) self.assertEqual(len(msgs), 2) self.assertEqual(msgs[0]["role"], "system") self.assertEqual(msgs[0]["content"], "Be helpful") def test_list_input_with_instructions_replaces_existing_system(self): handler = self._make_handler() - msgs = handler._input_to_messages( + msgs = handler._normalize_input( {"input": [{"role": "system", "content": "Old"}, {"role": "user", "content": "Hi"}], "instructions": "New"} ) self.assertEqual(len(msgs), 2) @@ -1018,7 +823,7 @@ def test_flat_content_list(self): {"type": "input_text", "text": "Hello"}, {"type": "input_image", "image_url": "https://example.com/img.jpg"}, ] - msgs = handler._input_to_messages({"input": flat_input}) + msgs = handler._normalize_input({"input": flat_input}) self.assertEqual(len(msgs), 1) self.assertEqual(msgs[0]["role"], "user") self.assertEqual(msgs[0]["content"], flat_input) @@ -1027,7 +832,7 @@ def test_flat_content_list_with_instructions(self): """Flat content list with instructions prepends a system message.""" handler = self._make_handler() flat_input = [{"type": "input_text", "text": "Hello"}] - msgs = handler._input_to_messages({"input": flat_input, "instructions": "Be brief"}) + msgs = handler._normalize_input({"input": flat_input, "instructions": "Be brief"}) self.assertEqual(len(msgs), 2) self.assertEqual(msgs[0], {"role": "system", "content": "Be brief"}) self.assertEqual(msgs[1]["role"], "user") @@ -1221,96 +1026,6 @@ def test_streaming_usage(self): self.assertGreater(usage.output_tokens, 0) self.assertEqual(usage.total_tokens, usage.input_tokens + usage.output_tokens) - def test_tool_call_streaming(self): - """Streaming responses with tools should emit function_call events.""" - tool_def = { - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - "description": "Get the weather for a city.", - }, - "type": "function", - } - events = list( - self.client.responses.create( - model=self.MODEL, - input="What is the weather in Paris?", - stream=True, - max_output_tokens=50, - tools=[tool_def], - ) - ) - types = [e.type for e in events] - self.assertIn("response.created", types) - self.assertIn("response.completed", types) - - # Should have function call events - self.assertIn("response.output_item.added", types) - self.assertIn("response.function_call_arguments.done", types) - - # Check the arguments done event - args_done = [e for e in events if e.type == "response.function_call_arguments.done"] - self.assertGreater(len(args_done), 0) - self.assertEqual(args_done[0].name, "get_weather") - - import json as json_mod - - parsed = json_mod.loads(args_done[0].arguments) - self.assertIsInstance(parsed, dict) - - def test_tool_call_non_streaming(self): - """Non-streaming responses with tools should include function_call output items.""" - tool_def = { - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - "description": "Get the weather for a city.", - }, - "type": "function", - } - resp = self.client.responses.create( - model=self.MODEL, - input="What is the weather in Paris?", - stream=False, - max_output_tokens=50, - tools=[tool_def], - ) - self.assertEqual(resp.status, "completed") - - # Should have at least message + function_call in output - self.assertGreater(len(resp.output), 1) - fc_items = [o for o in resp.output if o.type == "function_call"] - self.assertGreater(len(fc_items), 0) - self.assertEqual(fc_items[0].name, "get_weather") - - import json as json_mod - - parsed = json_mod.loads(fc_items[0].arguments) - self.assertIsInstance(parsed, dict) - - def test_tool_call_multi(self): - """Model should produce multiple tool calls when asked about two cities.""" - tool_def = { - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - "description": "Get the weather for a city.", - }, - "type": "function", - } - events = list( - self.client.responses.create( - model=self.MODEL, - input="What is the weather in Paris and London?", - stream=True, - max_output_tokens=100, - tools=[tool_def], - ) - ) - args_done = [e for e in events if e.type == "response.function_call_arguments.done"] - self.assertEqual(len(args_done), 2, f"Expected 2 tool calls, got {len(args_done)}") - self.assertEqual(events[-1].type, "response.completed") - def test_multi_turn(self): """Multi-turn conversation via list input.""" resp = self.client.responses.create( @@ -1872,6 +1587,377 @@ def test_responses_with_video_streaming(self): self._assert_video_description(text) +class TestToolCallUnit(unittest.TestCase): + """Unit tests for tool call parsing utilities (no server needed).""" + + def test_get_tool_call_config_fallback(self): + """Fallback config is returned for known model families (Qwen).""" + model = MagicMock() + model.config.model_type = "qwen2" + processor = MagicMock(spec=["convert_tokens_to_ids"]) + processor.convert_tokens_to_ids.return_value = 151657 + config = get_tool_call_config(processor, model) + self.assertIsNotNone(config) + self.assertEqual(config["stc_id"], 151657) + self.assertEqual(config["etc_id"], 151657) + + def test_get_tool_call_config_unsupported(self): + """None is returned for models without tool call support.""" + model = MagicMock() + model.config.model_type = "llama" + processor = MagicMock(spec=[]) + self.assertIsNone(get_tool_call_config(processor, model)) + + def test_parse_tool_calls_from_text(self): + text = '\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n' + processor = MagicMock() + processor.parse_response = lambda t, s: recursive_parse(t, s) + calls = parse_tool_calls(processor, text, _TOOL_CALL_FALLBACKS["qwen"]["schema"]) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["name"], "get_weather") + + def test_parse_multiple_tool_calls_from_text(self): + text = ( + '\n{"name": "get_weather", "arguments": {"city": "Paris"}}\n\n' + '\n{"name": "get_weather", "arguments": {"city": "London"}}\n' + ) + processor = MagicMock() + processor.parse_response = lambda t, s: recursive_parse(t, s) + calls = parse_tool_calls(processor, text, _TOOL_CALL_FALLBACKS["qwen"]["schema"]) + self.assertEqual(len(calls), 2) + + +class _TestToolCallBase: + """Base class for tool call integration tests. Subclasses set MODEL and inherit all tests.""" + + MODEL: str + + @classmethod + def setUpClass(cls): + cls.serve, port = _start_serve() + cls.base_url = f"http://localhost:{port}" + cls.client = OpenAI(base_url=f"{cls.base_url}/v1", api_key="unused") + + @classmethod + def tearDownClass(cls): + cls.serve.kill_server() + + def _get_tool_def(self): + return { + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + "description": "Get the weather for a city.", + }, + "type": "function", + } + + def test_chat_non_streaming(self): + resp = self.client.chat.completions.create( + model=self.MODEL, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + stream=False, + max_tokens=50, + temperature=0.0, + tools=[self._get_tool_def()], + ) + self.assertEqual(resp.choices[0].finish_reason, "tool_calls") + self.assertIsNotNone(resp.choices[0].message.tool_calls) + tc = resp.choices[0].message.tool_calls[0] + self.assertEqual(tc.function.name, "get_weather") + parsed_args = json.loads(tc.function.arguments) + self.assertIsInstance(parsed_args, dict) + + def test_chat_streaming(self): + chunks = list( + self.client.chat.completions.create( + model=self.MODEL, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + stream=True, + max_tokens=50, + temperature=0.0, + tools=[self._get_tool_def()], + ) + ) + tool_chunks = [c for c in chunks if c.choices[0].delta.tool_calls] + self.assertGreater(len(tool_chunks), 0, "Model did not produce a tool call") + first_tool = tool_chunks[0].choices[0].delta.tool_calls[0] + self.assertEqual(first_tool.function.name, "get_weather") + self.assertEqual(chunks[-1].choices[0].finish_reason, "tool_calls") + parsed_args = json.loads(first_tool.function.arguments) + self.assertIsInstance(parsed_args, dict) + + def test_chat_multiple_tool_calls_non_streaming(self): + resp = self.client.chat.completions.create( + model=self.MODEL, + messages=[{"role": "user", "content": "What is the weather in Paris and London?"}], + stream=False, + max_tokens=100, + temperature=0.0, + tools=[self._get_tool_def()], + ) + self.assertEqual(resp.choices[0].finish_reason, "tool_calls") + self.assertEqual(len(resp.choices[0].message.tool_calls), 2) + + def test_chat_multiple_tool_calls_streaming(self): + chunks = list( + self.client.chat.completions.create( + model=self.MODEL, + messages=[{"role": "user", "content": "What is the weather in Paris and London?"}], + stream=True, + max_tokens=100, + temperature=0.0, + tools=[self._get_tool_def()], + ) + ) + tool_chunks = [c for c in chunks if c.choices[0].delta.tool_calls] + self.assertEqual(len(tool_chunks), 2, f"Expected 2 tool calls, got {len(tool_chunks)}") + self.assertEqual(chunks[-1].choices[0].finish_reason, "tool_calls") + + def test_chat_multi_turn_non_streaming(self): + tool_def = self._get_tool_def() + resp1 = self.client.chat.completions.create( + model=self.MODEL, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + stream=False, + max_tokens=50, + temperature=0.0, + tools=[tool_def], + ) + self.assertEqual(resp1.choices[0].finish_reason, "tool_calls") + tc = resp1.choices[0].message.tool_calls[0] + + resp2 = self.client.chat.completions.create( + model=self.MODEL, + messages=[ + {"role": "user", "content": "What is the weather in Paris?"}, + resp1.choices[0].message, + {"role": "tool", "tool_call_id": tc.id, "content": '{"temperature": 22, "condition": "sunny"}'}, + ], + stream=False, + max_tokens=100, + temperature=0.0, + tools=[tool_def], + ) + self.assertIn(resp2.choices[0].finish_reason, ("stop", "length")) + content = resp2.choices[0].message.content + self.assertIsNotNone(content) + self.assertTrue( + "22" in content.lower() or "sunny" in content.lower(), + f"Expected model to reference tool result, got: {content}", + ) + + def test_chat_multi_turn_streaming(self): + tool_def = self._get_tool_def() + + # Turn 1: streaming — accumulate tool call from deltas + chunks = list( + self.client.chat.completions.create( + model=self.MODEL, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + stream=True, + max_tokens=50, + temperature=0.0, + tools=[tool_def], + ) + ) + self.assertEqual(chunks[-1].choices[0].finish_reason, "tool_calls") + tool_chunks = [c for c in chunks if c.choices[0].delta.tool_calls] + self.assertGreater(len(tool_chunks), 0) + tc = tool_chunks[0].choices[0].delta.tool_calls[0] + + # Reconstruct assistant message from deltas + content = "".join(c.choices[0].delta.content for c in chunks if c.choices[0].delta.content) + assistant_msg = { + "role": "assistant", + "content": content, + "tool_calls": [{"id": tc.id, "type": "function", "function": tc.function.model_dump()}], + } + + # Turn 2: streaming — send back tool result + chunks2 = list( + self.client.chat.completions.create( + model=self.MODEL, + messages=[ + {"role": "user", "content": "What is the weather in Paris?"}, + assistant_msg, + {"role": "tool", "tool_call_id": tc.id, "content": '{"temperature": 22, "condition": "sunny"}'}, + ], + stream=True, + max_tokens=100, + temperature=0.0, + tools=[tool_def], + ) + ) + content = "".join(c.choices[0].delta.content for c in chunks2 if c.choices[0].delta.content) + self.assertTrue( + "22" in content.lower() or "sunny" in content.lower(), + f"Expected model to reference tool result, got: {content}", + ) + + def test_responses_non_streaming(self): + resp = self.client.responses.create( + model=self.MODEL, + input="What is the weather in Paris?", + stream=False, + max_output_tokens=50, + tools=[self._get_tool_def()], + ) + self.assertEqual(resp.status, "completed") + fc_items = [o for o in resp.output if o.type == "function_call"] + self.assertGreater(len(fc_items), 0) + self.assertEqual(fc_items[0].name, "get_weather") + parsed = json.loads(fc_items[0].arguments) + self.assertIsInstance(parsed, dict) + + def test_responses_streaming(self): + events = list( + self.client.responses.create( + model=self.MODEL, + input="What is the weather in Paris?", + stream=True, + max_output_tokens=50, + tools=[self._get_tool_def()], + ) + ) + types = [e.type for e in events] + self.assertIn("response.created", types) + self.assertIn("response.completed", types) + self.assertIn("response.function_call_arguments.done", types) + + args_done = [e for e in events if e.type == "response.function_call_arguments.done"] + self.assertGreater(len(args_done), 0) + self.assertEqual(args_done[0].name, "get_weather") + parsed = json.loads(args_done[0].arguments) + self.assertIsInstance(parsed, dict) + + def test_responses_multiple_tool_calls_non_streaming(self): + resp = self.client.responses.create( + model=self.MODEL, + input="What is the weather in Paris and London?", + stream=False, + max_output_tokens=100, + tools=[self._get_tool_def()], + ) + self.assertEqual(resp.status, "completed") + fc_items = [o for o in resp.output if o.type == "function_call"] + self.assertEqual(len(fc_items), 2, f"Expected 2 tool calls, got {len(fc_items)}") + + def test_responses_multiple_tool_calls_streaming(self): + events = list( + self.client.responses.create( + model=self.MODEL, + input="What is the weather in Paris and London?", + stream=True, + max_output_tokens=100, + tools=[self._get_tool_def()], + ) + ) + args_done = [e for e in events if e.type == "response.function_call_arguments.done"] + self.assertEqual(len(args_done), 2, f"Expected 2 tool calls, got {len(args_done)}") + self.assertEqual(events[-1].type, "response.completed") + + def test_responses_multi_turn_non_streaming(self): + tool_def = self._get_tool_def() + resp1 = self.client.responses.create( + model=self.MODEL, + input="What is the weather in Paris?", + stream=False, + max_output_tokens=50, + tools=[tool_def], + ) + self.assertEqual(resp1.status, "completed") + fc_items = [o for o in resp1.output if o.type == "function_call"] + self.assertGreater(len(fc_items), 0) + + input_list = [{"role": "user", "content": "What is the weather in Paris?"}] + input_list += resp1.output + input_list.append( + { + "type": "function_call_output", + "call_id": fc_items[0].call_id, + "output": '{"temperature": 22, "condition": "sunny"}', + } + ) + resp2 = self.client.responses.create( + model=self.MODEL, + input=input_list, + stream=False, + max_output_tokens=100, + tools=[tool_def], + ) + self.assertEqual(resp2.status, "completed") + msg_items = [o for o in resp2.output if o.type == "message"] + self.assertGreater(len(msg_items), 0) + content = msg_items[0].content[0].text + self.assertTrue( + "22" in content.lower() or "sunny" in content.lower(), + f"Expected model to reference tool result, got: {content}", + ) + + def test_responses_multi_turn_streaming(self): + tool_def = self._get_tool_def() + + # Turn 1: streaming — get completed response with tool calls + events = list( + self.client.responses.create( + model=self.MODEL, + input="What is the weather in Paris?", + stream=True, + max_output_tokens=50, + tools=[tool_def], + ) + ) + completed = [e for e in events if e.type == "response.completed"] + self.assertEqual(len(completed), 1) + resp1_output = completed[0].response.output + fc_items = [o for o in resp1_output if o.type == "function_call"] + self.assertGreater(len(fc_items), 0) + + # Turn 2: streaming — send back tool result + input_list = [{"role": "user", "content": "What is the weather in Paris?"}] + input_list += resp1_output + input_list.append( + { + "type": "function_call_output", + "call_id": fc_items[0].call_id, + "output": '{"temperature": 22, "condition": "sunny"}', + } + ) + events2 = list( + self.client.responses.create( + model=self.MODEL, + input=input_list, + stream=True, + max_output_tokens=100, + tools=[tool_def], + ) + ) + content = "".join(e.delta for e in events2 if e.type == "response.output_text.delta") + self.assertTrue( + "22" in content.lower() or "sunny" in content.lower(), + f"Expected model to reference tool result, got: {content}", + ) + + +@slow +@require_serve +@require_torch_accelerator +class TestToolCallQwen(_TestToolCallBase, unittest.TestCase): + """Tool call tests with Qwen (fallback config, no response_schema).""" + + MODEL = "Qwen/Qwen2.5-0.5B-Instruct" + + +@slow +@require_serve +@require_torch_accelerator +class TestToolCallGemma(_TestToolCallBase, unittest.TestCase): + """Tool call tests with Gemma 4 (response_schema + stc/etc special tokens).""" + + MODEL = "google/gemma-4-E2B-it" + + @slow @require_librosa @require_multipart diff --git a/tests/generation/test_configuration_utils.py b/tests/generation/test_configuration_utils.py index 3ca904db0c57..36ddf4844d54 100644 --- a/tests/generation/test_configuration_utils.py +++ b/tests/generation/test_configuration_utils.py @@ -157,31 +157,47 @@ def test_validate(self): GenerationConfig() self.assertEqual(len(captured_logs.out), 0) - # Inconsequent but technically wrong configuration will throw a warning (e.g. setting sampling - # parameters with `do_sample=False`). May be escalated to an error in the future. + # Inconsequent but technically wrong configuration will throw a warning (e.g. requesting an extra output + # without `return_dict_in_generate=True`). May be escalated to an error in the future. logger.warning_once.cache_clear() with CaptureLogger(logger) as captured_logs: GenerationConfig(return_dict_in_generate=False, output_scores=True) self.assertNotEqual(len(captured_logs.out), 0) + # Explicitly setting a sampling flag alongside `do_sample=False` still warns: this is a user-level mistake. logger.warning_once.cache_clear() with CaptureLogger(logger) as captured_logs: generation_config_bad_temperature = GenerationConfig(do_sample=False, temperature=0.5) # store for later self.assertNotEqual(len(captured_logs.out), 0) - # Expanding on the case above, we can update a bad configuration to get rid of the warning. Ideally, - # that is done by unsetting the parameter (i.e. setting it to None) + # But a value inherited from a model's default config (i.e. not in this update's kwargs) does NOT warn: in + # the real world, `generate(do_sample=False)` on a model whose `generation_config.json` has `temperature=0.6` + # would otherwise log a useless warning. + logger.warning_once.cache_clear() + base_config = GenerationConfig(do_sample=True, temperature=0.6) # mimics a model's default config + with CaptureLogger(logger) as captured_logs: + base_config.update(do_sample=False) + self.assertEqual(len(captured_logs.out), 0) + + # Inverse provenance case: `do_sample=False` inherited from a model's config (so not user-set this call), user only + # sets a sampling flag. The conflict SHOULD produce noise because the user may think that it's non-greedy by default + logger.warning_once.cache_clear() + greedy_hub_config = GenerationConfig(do_sample=False) # mimics a model's default config forcing greedy + with CaptureLogger(logger) as captured_logs: + greedy_hub_config.update(top_p=0.8) + self.assertNotEqual(len(captured_logs.out), 0) + + # Updating only `temperature` (do_sample was pre-existing, i.e. "from the hub") does warn logger.warning_once.cache_clear() with CaptureLogger(logger) as captured_logs: - # BAD - 0.9 means it is still set, we should warn generation_config_bad_temperature.update(temperature=0.9) self.assertNotEqual(len(captured_logs.out), 0) + # But setting both in the same `update()` call DOES warn. logger.warning_once.cache_clear() with CaptureLogger(logger) as captured_logs: - # CORNER CASE - 1.0 is the default, we can't detect whether it is set by the user or not, we shouldn't warn - generation_config_bad_temperature.update(temperature=1.0) - self.assertEqual(len(captured_logs.out), 0) + generation_config_bad_temperature.update(do_sample=False, temperature=0.9) + self.assertNotEqual(len(captured_logs.out), 0) logger.warning_once.cache_clear() with CaptureLogger(logger) as captured_logs: @@ -230,6 +246,63 @@ def test_validate(self): with self.assertRaises(ValueError): generation_config.validate(strict=True) + def test_validate_sampling_flag_provenance(self): + """ + Dedicated coverage for the provenance-aware warning rule on sampling-only flags: + we only warn when BOTH `do_sample=False` AND a conflicting sampling flag (e.g. `top_p`, `temperature`) + were explicitly provided by the caller in the same context, or none of the 2 were directly provided, or only + the sampling flag is provided along do_sample=False already existing. + """ + logger = transformers_logging.get_logger("transformers.generation.configuration_utils") + + def _warn_count(fn): + logger.warning_once.cache_clear() + with CaptureLogger(logger) as captured: + fn() + return len(captured.out) + + # 1. Hub config sets `temperature`, user does only `generate(do_sample=False)` -> NO warning. + # (Emulates: model whose `generation_config.json` carries `do_sample=True, temperature=0.6`, user + # explicitly asks for greedy decoding.) + def case_hub_temp_user_do_sample_only(): + cfg = GenerationConfig(do_sample=True, temperature=0.6) # stands in for the hub default + cfg.update(do_sample=False) + + self.assertEqual(_warn_count(case_hub_temp_user_do_sample_only), 0) + + # 2. User explicitly sets BOTH `do_sample=False` and `top_p=0.8` in the same call -> WARN. + self.assertNotEqual(_warn_count(lambda: GenerationConfig(do_sample=False, top_p=0.8)), 0) + + # 3. User explicitly sets only `do_sample=False` (no sampling flag) -> NO warning, even though + # attribute defaults (like `top_k=50`) may be present. + self.assertEqual(_warn_count(lambda: GenerationConfig(do_sample=False)), 0) + + # 4. Hub config forces greedy (`do_sample=False`), user sets only `top_p=0.8` -> warnings: + # do_sample` was inherited, but clashes with user-expressed intent, so flagging their `top_p` + def case_hub_greedy_user_top_p(): + cfg = GenerationConfig(do_sample=False) # stands in for the hub default + cfg.update(top_p=0.8) + + self.assertNotEqual(_warn_count(case_hub_greedy_user_top_p), 0) + + # 5. User sets `do_sample=False` and `temperature=0.5` via a single `update()` call -> WARN. + def case_update_both_sides(): + cfg = GenerationConfig() + cfg.update(do_sample=False, temperature=0.5) + + self.assertNotEqual(_warn_count(case_update_both_sides), 0) + + # 6. Same idea for beam flags: user only asks for `num_beams=1`, hub default has `length_penalty=0.8` + # -> NO warning. + def case_hub_length_penalty_user_num_beams_only(): + cfg = GenerationConfig(num_beams=4, length_penalty=0.8) # stands in for the hub default + cfg.update(num_beams=1) + + self.assertEqual(_warn_count(case_hub_length_penalty_user_num_beams_only), 0) + + # 7. User sets BOTH `num_beams=1` and `length_penalty=0.8` explicitly -> WARN. + self.assertNotEqual(_warn_count(lambda: GenerationConfig(num_beams=1, length_penalty=0.8)), 0) + def test_refuse_to_save(self): """Tests that we refuse to save a generation config that fails validation.""" diff --git a/tests/models/auto/test_tokenization_auto.py b/tests/models/auto/test_tokenization_auto.py index 0d8c099ca097..5e584a55b21f 100644 --- a/tests/models/auto/test_tokenization_auto.py +++ b/tests/models/auto/test_tokenization_auto.py @@ -58,6 +58,7 @@ SMALL_MODEL_IDENTIFIER, CaptureLogger, RequestCounter, + require_sentencepiece, require_tokenizers, slow, ) @@ -746,3 +747,21 @@ def __init__(self, **kwargs): self.assertTrue(tokenizer.special_attribute_present) finally: os.chdir(prev_dir) + + @require_tokenizers + @require_sentencepiece + def test_mismatched_model_type_uses_config_tokenizer_class_with_sentencepiece(self): + tokenizer = AutoTokenizer.from_pretrained( + "facebook/nllb-200-distilled-600M", + revision="f8d333a098d19b4fd9a8b18f94170487ad3f821d", + ) + self.assertEqual(tokenizer.__class__.__name__, "NllbTokenizer") + + @require_tokenizers + def test_mismatched_model_type_uses_config_tokenizer_class_without_sentencepiece(self): + with mock.patch("transformers.models.auto.tokenization_auto.is_sentencepiece_available", return_value=False): + tokenizer = AutoTokenizer.from_pretrained( + "facebook/nllb-200-distilled-600M", + revision="f8d333a098d19b4fd9a8b18f94170487ad3f821d", + ) + self.assertEqual(tokenizer.__class__.__name__, "NllbTokenizer") diff --git a/tests/models/granite_speech/test_modeling_granite_speech.py b/tests/models/granite_speech/test_modeling_granite_speech.py index c5e7aa3defcd..95c6c443d6f0 100644 --- a/tests/models/granite_speech/test_modeling_granite_speech.py +++ b/tests/models/granite_speech/test_modeling_granite_speech.py @@ -230,6 +230,12 @@ def setUp(self): has_text_modality=False, ) + @unittest.skip( + reason="This test does not apply to GraniteSpeech since inputs_embeds corresponding to audio tokens are replaced when input features are provided." + ) + def test_inputs_embeds_matches_input_ids(self): + pass + def test_inputs_embeds(self): # overwrite inputs_embeds tests because we need to delete "input features" for the audio model config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/models/modernbert/test_modeling_modernbert.py b/tests/models/modernbert/test_modeling_modernbert.py index 7f7b5fa87f15..db01881aa678 100644 --- a/tests/models/modernbert/test_modeling_modernbert.py +++ b/tests/models/modernbert/test_modeling_modernbert.py @@ -428,6 +428,10 @@ def test_inference_masked_lm_flash_attention_2(self): [[[3.8203, -0.2125, 12.2812], [3.6055, 0.6797, 14.6875], [-5.1094, -3.8105, 11.9922]]], dtype=torch.float16, ), + ("rocm", None): torch.tensor( + [[[3.8223, -0.2045, 12.2891], [3.6328, 0.6875, 14.7031], [-5.1133, -3.8105, 11.9922]]], + dtype=torch.float16, + ), ("xpu", None): torch.tensor( [[[3.8555, -0.1993, 12.2969], [3.6387, 0.6943, 14.7109], [-5.1172, -3.8086, 11.9844]]], dtype=torch.float16, diff --git a/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py b/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py index 327bf75bbbc8..5a425b434e7d 100644 --- a/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py +++ b/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py @@ -545,18 +545,10 @@ def test_small_model_integration_test_batch(self): # it should not matter whether two images are the same size or not output = model.generate(**inputs, max_new_tokens=30, do_sample=False) - expected_decoded_texts = Expectations( - { - (None, None): [ - 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in', - 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in', - ], - ("rocm", (9, 4)): [ - 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is often reflected', - 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is often reflected' - ], - } - ).get_expectation() # fmt: skip + expected_decoded_texts = [ + "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in", + "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in", + ] self.assertEqual( self.processor.batch_decode(output, skip_special_tokens=True), @@ -609,7 +601,7 @@ def test_small_model_integration_test_batch_wo_image(self): 'system\nYou are a helpful assistant.\nuser\nWho are you?\nassistant\nI am Qwen, an AI language model created by Alibaba Cloud. I am designed to assist with various tasks such as answering questions, providing information,' ], ("rocm", (9, 4)): [ - 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is evident in', + 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in', 'system\nYou are a helpful assistant.\nuser\nWho are you?\nassistant\nI am Qwen, a large language model created by Alibaba Cloud. I am designed to assist with a wide range of tasks, from answering questions and' ], ("xpu", None): [ @@ -653,8 +645,8 @@ def test_small_model_integration_test_batch_different_resolutions(self): 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is evident in', ], ("rocm", None): [ - 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is often reflected', - 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is evident in' + 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in', + 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is evident in', ], ("xpu", None): [ 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in', @@ -690,11 +682,7 @@ def test_small_model_integration_test_batch_flashatt2(self): # it should not matter whether two images are the same size or not output = model.generate(**inputs, max_new_tokens=30, do_sample=False) - expected_decoded_text = Expectations({ - ("cuda", None): "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in", - ("rocm", (9, 4)): "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and gentle nature, which is evident in", - ("xpu", None): "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in", - }).get_expectation() # fmt: skip + expected_decoded_text = "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in" # Since the test is to generate twice the same text, we just test twice against the expected decoded text decoded_texts = self.processor.batch_decode(output, skip_special_tokens=True) @@ -776,9 +764,6 @@ def test_small_model_integration_test_with_video(self): (None, None): [ 'system\nYou are a helpful assistant.\nuser\nWhat is shown in this video?\nassistant\nThe video shows two individuals playing tennis on an indoor court. The player in the foreground, dressed in a white shirt and black shorts, is preparing to', ], - ("rocm", (9, 4)): [ - 'system\nYou are a helpful assistant.\nuser\nWhat is shown in this video?\nassistant\nThe video shows an indoor tennis court with a person standing on the service line, preparing to serve. The individual appears to be practicing or warming up,', - ], ("xpu", None): [ 'system\nYou are a helpful assistant.\nuser\nWhat is shown in this video?\nassistant\nThe video shows an indoor tennis court with a person standing on the service line, preparing to serve. The individual appears to be practicing or warming up,', ], diff --git a/tests/models/sam3_lite_text/test_modeling_sam3_lite_text.py b/tests/models/sam3_lite_text/test_modeling_sam3_lite_text.py index 05a9307bfa87..c9b0766dc5d1 100644 --- a/tests/models/sam3_lite_text/test_modeling_sam3_lite_text.py +++ b/tests/models/sam3_lite_text/test_modeling_sam3_lite_text.py @@ -661,6 +661,13 @@ def test_sdpa_can_compile_dynamic(self): def test_sdpa_can_dispatch_on_flash(self): pass + @unittest.skip( + reason="Sam3LiteTextModel creates float attention masks from features (with gradients) in the DETR " + "encoder/decoder, which Flash Attention requires to be None." + ) + def test_flash_attn_2_can_dispatch_composite_models(self): + pass + def test_model_outputs_equivalence(self): """ Test that tuple and dict outputs are equivalent. diff --git a/tests/quantization/ggml/test_ggml.py b/tests/quantization/ggml/test_ggml.py index 763f8ac40502..aa5cdbc7adc6 100644 --- a/tests/quantization/ggml/test_ggml.py +++ b/tests/quantization/ggml/test_ggml.py @@ -351,6 +351,8 @@ class GgufModelTests(unittest.TestCase): q4_k_m_qwen3moe_model_id = "Qwen3-30B-A3B-Q4_K_M.gguf" q8_0_umt5_encoder_model_id = "umt5-xxl-encoder-Q8_0.gguf" q4_k_m_lfm2_model_id = "LFM2-1.2B-Q4_K_M.gguf" + gpt_oss_model_id = "unsloth/gpt-oss-20b-GGUF" + gpt_oss_gguf_file = "gpt-oss-20b-Q5_K_M.gguf" example_text = "Hello" @@ -384,6 +386,20 @@ def test_qwen2_q4_0(self): EXPECTED_TEXT = "Hello.jsoup\n\nI am a beginner" self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT) + def test_gpt_oss_q5_k_m(self): + tokenizer = AutoTokenizer.from_pretrained(self.gpt_oss_model_id, gguf_file=self.gpt_oss_gguf_file) + model = AutoModelForCausalLM.from_pretrained( + self.gpt_oss_model_id, + gguf_file=self.gpt_oss_gguf_file, + device_map="auto", + dtype=torch.float16, + ) + + text = tokenizer(self.example_text, return_tensors="pt").to(torch_device) + out = model.generate(**text, max_new_tokens=10) + EXPECTED_TEXT = "Hello, I just want to say that I am just" + self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT) + def test_qwen2moe_q8(self): tokenizer = AutoTokenizer.from_pretrained(self.qwen2moe_model_id, gguf_file=self.q8_qwen2moe_model_id) model = AutoModelForCausalLM.from_pretrained( diff --git a/tests/repo_utils/test_check_modular_conversion.py b/tests/repo_utils/test_check_modular_conversion.py new file mode 100644 index 000000000000..f75b53174480 --- /dev/null +++ b/tests/repo_utils/test_check_modular_conversion.py @@ -0,0 +1,74 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest +from unittest.mock import patch + + +git_repo_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +utils_path = os.path.join(git_repo_path, "utils") +if utils_path not in sys.path: + sys.path.append(utils_path) + +import check_modular_conversion # noqa: E402 + + +class ConverterChangedInDiffTest(unittest.TestCase): + """Regression guard for PR #45492: changes to the converter alone must force a full check.""" + + def _patch_modified(self, files): + return patch.object(check_modular_conversion, "_get_modified_files", return_value=files) + + def test_returns_true_when_modular_model_converter_changed(self): + with self._patch_modified( + [ + "utils/modular_model_converter.py", + "src/transformers/models/llava_onevision/modular_llava_onevision.py", + ] + ): + self.assertTrue(check_modular_conversion.converter_changed_in_diff()) + + def test_returns_true_when_create_dependency_mapping_changed(self): + with self._patch_modified(["utils/create_dependency_mapping.py"]): + self.assertTrue(check_modular_conversion.converter_changed_in_diff()) + + def test_returns_false_for_model_only_diff(self): + with self._patch_modified( + [ + "src/transformers/models/llama/modular_llama.py", + "src/transformers/models/llama/modeling_llama.py", + ] + ): + self.assertFalse(check_modular_conversion.converter_changed_in_diff()) + + def test_returns_false_for_unrelated_utils_change(self): + with self._patch_modified(["utils/check_modular_conversion.py", "utils/check_copies.py"]): + self.assertFalse(check_modular_conversion.converter_changed_in_diff()) + + def test_converter_files_set_includes_expected_entries(self): + # Keep the allow-list grounded: if either file is renamed/removed, this test fails loudly + # so the detection logic is updated alongside the rename. + self.assertIn("utils/modular_model_converter.py", check_modular_conversion.CONVERTER_FILES) + self.assertIn("utils/create_dependency_mapping.py", check_modular_conversion.CONVERTER_FILES) + for rel_path in check_modular_conversion.CONVERTER_FILES: + self.assertTrue( + os.path.exists(os.path.join(git_repo_path, rel_path)), + f"{rel_path} listed in CONVERTER_FILES but does not exist on disk", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tensor_parallel_mixin.py b/tests/test_tensor_parallel_mixin.py index 4e4e60159eec..68780cf2f8ab 100644 --- a/tests/test_tensor_parallel_mixin.py +++ b/tests/test_tensor_parallel_mixin.py @@ -16,6 +16,7 @@ from abc import ABC, abstractmethod from transformers import TorchAoConfig, set_seed +from transformers.distributed.configuration_utils import DistributedConfig from transformers.integrations.tensor_parallel import _get_parameter_tp_plan from transformers.testing_utils import ( is_tensor_parallel_test, @@ -344,6 +345,72 @@ def _test_tp_generation_quantized_impl(_rank, model_path, model_class, max_new_t dist.barrier() +def _load_ep_and_reference_models(model_path, model_class): + """Load EP model and non-EP reference model for comparison.""" + model_ep = model_class.from_pretrained( + model_path, + distributed_config=DistributedConfig(enable_expert_parallel=True), + ) + dist.barrier() + + device = model_ep.device + model_ref = model_class.from_pretrained(model_path) + model_ref = model_ref.to(device) + + return model_ep, model_ref, device + + +def _test_ep_forward_impl(_rank, model_path, model_class, atol, rtol): + """Implementation for comparing EP and non-EP model outputs.""" + set_seed(0) + + model_ep, model_ref, device = _load_ep_and_reference_models(model_path, model_class) + + model_ep.eval() + model_ref.eval() + + vocab_size = model_ref.config.vocab_size + input_ids = torch.randint(0, vocab_size, (2, 64)).to(device) + + with torch.no_grad(): + logits_ref = model_ref(input_ids).logits + logits_ep = model_ep(input_ids).logits + + diff = (logits_ref - logits_ep).abs() + assert torch.allclose(logits_ref, logits_ep, atol=atol, rtol=rtol), ( + f"EP and non-EP model outputs differ. Max diff: {diff.max().item()} | Min diff: {diff.min().item()}" + ) + + dist.barrier() + + +def _test_ep_backward_impl(_rank, model_path, model_class, atol, rtol): + """Implementation for comparing EP and non-EP model backward passes.""" + set_seed(0) + + model_ep, model_ref, device = _load_ep_and_reference_models(model_path, model_class) + model_ep.train() + model_ref.train() + + vocab_size = model_ref.config.vocab_size + input_ids = torch.randint(0, vocab_size, (2, 64)).to(device) + labels = torch.randint(0, vocab_size, (2, 64)).to(device) + + loss_ref = model_ref(input_ids, labels=labels).loss + loss_ref.backward() + + loss_ep = model_ep(input_ids, labels=labels).loss + loss_ep.backward() + + assert torch.allclose(loss_ref, loss_ep, atol=atol, rtol=rtol), ( + f"EP and non-EP model losses differ. " + f"Non-EP loss: {loss_ref.item()}, EP loss: {loss_ep.item()}, " + f"Diff: {(loss_ref - loss_ep).abs().item()}" + ) + + dist.barrier() + + class TensorParallelTesterMixin(ABC): """ Mixin for tensor parallel tests. Add to model test classes alongside ModelTesterMixin. @@ -371,6 +438,11 @@ def model_tester(self): # ============================================================ # Helper methods # ============================================================ + def _has_ep_plan(self) -> bool: + """Check if model has an expert parallel plan defined.""" + config = self.model_tester.get_config() + return hasattr(config, "base_model_ep_plan") and config.base_model_ep_plan is not None + def _has_tp_plan(self) -> bool: """Check if model has a tensor parallel plan defined.""" config = self.model_tester.get_config() @@ -411,6 +483,26 @@ def _skip_if_not_supported(self): # if hasattr(config, "vision_config") and config.vision_config is not None: # self.skipTest("VLM models are not yet supported in TP tests") + def _skip_if_ep_not_supported(self): + """Check and skip test if EP is not supported for this model/environment.""" + if not is_torch_greater_or_equal("2.9"): + self.skipTest("Expert parallel tests require torch >= 2.9") + + if torch.cuda.is_available() or torch.xpu.is_available(): + self.skipTest("Expert parallel mixin tests are CPU-only and should not run on GPU or XPU machines") + + if os.cpu_count() < self.tensor_parallel_size: + self.skipTest( + f"Expert parallel tests require at least {self.tensor_parallel_size} CPUs, " + f"but only {os.cpu_count()} available" + ) + + if not hasattr(self.model_tester, "causal_lm_class") or self.model_tester.causal_lm_class is None: + self.skipTest("Model tester does not have causal_lm_class (not using CausalLMModelTester)") + + if not self._has_ep_plan(): + self.skipTest("Model does not have an expert parallel plan (base_model_ep_plan)") + @is_tensor_parallel_test def test_tp_forward(self): self._skip_if_not_supported() @@ -482,3 +574,35 @@ def test_tp_generation_quantized(self): _init_distributed(tp=self.tensor_parallel_size)(_test_tp_generation_quantized_impl)( tmp_dir, model_class, max_new_tokens ) + + @is_tensor_parallel_test + def test_ep_forward(self): + self._skip_if_ep_not_supported() + + config = self.model_tester.get_config() + model_class = self._get_tp_model_class() + atol = self.tensor_parallel_atol + rtol = self.tensor_parallel_rtol + + with tempfile.TemporaryDirectory() as tmp_dir: + set_seed(42) + model = model_class(config) + model.save_pretrained(tmp_dir, save_original_format=True) + + _init_distributed(tp=self.tensor_parallel_size)(_test_ep_forward_impl)(tmp_dir, model_class, atol, rtol) + + @is_tensor_parallel_test + def test_ep_backward(self): + self._skip_if_ep_not_supported() + + config = self.model_tester.get_config() + model_class = self._get_tp_model_class() + atol = self.tensor_parallel_atol + rtol = self.tensor_parallel_rtol + + with tempfile.TemporaryDirectory() as tmp_dir: + set_seed(42) + model = model_class(config) + model.save_pretrained(tmp_dir, save_original_format=True) + + _init_distributed(tp=self.tensor_parallel_size)(_test_ep_backward_impl)(tmp_dir, model_class, atol, rtol) diff --git a/tests/tokenization/test_tokenization_utils.py b/tests/tokenization/test_tokenization_utils.py index da02adcc484d..1b91903efec7 100644 --- a/tests/tokenization/test_tokenization_utils.py +++ b/tests/tokenization/test_tokenization_utils.py @@ -352,3 +352,24 @@ def test_special_tokens_overwrite(self): new_tokenizer.decode(new_tokenizer.encode(text_with_nonspecial_tokens), skip_special_tokens=True) == text_with_nonspecial_tokens ) + + def test_import_protobuf_decode_error_without_protobuf(self): + from unittest.mock import patch + + from transformers.tokenization_utils_base import import_protobuf_decode_error + + with patch("transformers.tokenization_utils_base.is_protobuf_available", return_value=False): + result = import_protobuf_decode_error() + self.assertEqual(result, ()) + + def test_import_protobuf_decode_error_does_not_mask_exceptions(self): + from unittest.mock import patch + + from transformers.tokenization_utils_base import import_protobuf_decode_error + + with patch("transformers.tokenization_utils_base.is_protobuf_available", return_value=False): + with self.assertRaises(ValueError): + try: + raise ValueError("real error") + except import_protobuf_decode_error(): + pass diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index d31af329134b..603630bdb458 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -139,6 +139,47 @@ def test_tf32(self): self.check_trained_model(trainer.model) +# --------------------------------------------------------------------------- +# DDP kwargs forwarding tests +# --------------------------------------------------------------------------- + + +@require_torch +class TrainerDDPKwargsTest(TestCasePlus): + """The `ddp_*` TrainingArguments fields must reach DistributedDataParallelKwargs.""" + + def _get_ddp_kwargs(self, **training_args_overrides): + """Build a Trainer, run _build_accelerator_args, return the DDP kwargs dict.""" + with tempfile.TemporaryDirectory() as tmp_dir: + args = TrainingArguments(output_dir=tmp_dir, max_steps=1, **training_args_overrides) + trainer = Trainer(model=RegressionModel(), args=args, train_dataset=RegressionDataset()) + accelerator_args = trainer._build_accelerator_args() + (handler,) = accelerator_args["kwargs_handlers"] + return handler + + def test_ddp_static_graph_true_reaches_accelerator(self): + """ddp_static_graph=True is forwarded as static_graph=True to DistributedDataParallelKwargs.""" + handler = self._get_ddp_kwargs(ddp_static_graph=True) + self.assertTrue(handler.static_graph) + + def test_ddp_static_graph_false_reaches_accelerator(self): + """ddp_static_graph=False is forwarded as static_graph=False.""" + handler = self._get_ddp_kwargs(ddp_static_graph=False) + self.assertFalse(handler.static_graph) + + def test_ddp_static_graph_none_preserves_default(self): + """ddp_static_graph=None (default) must NOT override DistributedDataParallelKwargs' own default (False). + + Regression guard: the conditional in _build_accelerator_args must keep static_graph out of ddp_kwargs + when the flag is unset, otherwise clusters not configured for it would silently switch behavior. + """ + handler = self._get_ddp_kwargs() # ddp_static_graph unset + # DistributedDataParallelKwargs default is False. If our conditional is broken and we always injected + # the attribute, this would still be False only by coincidence. Cross-check with ddp_static_graph=True + # (above) that the kwarg IS plumbed when set — together these tests pin both directions. + self.assertFalse(handler.static_graph) + + # --------------------------------------------------------------------------- # Gradient accumulation tests # --------------------------------------------------------------------------- diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index 7366845c4d78..6a27b6b5e0fb 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -3451,3 +3451,67 @@ def test_vision_language_model(self): assert image_encoder is model.model.vision_tower, ( f"LLaVA get_encoder(modality='image') should return vision_tower, got {type(image_encoder)}" ) + + +@require_torch +class DisableMmapLoadingTest(unittest.TestCase): + """Tests for the `disable_mmap` kwarg in `load_state_dict` and the `_is_on_hf_mount` helper.""" + + def _fake_open_factory(self, proc_mounts_contents): + """Return a patched `open` that serves `proc_mounts_contents` for `/proc/mounts` and defers otherwise.""" + import builtins + + real_open = builtins.open + + def fake_open(path, *args, **kwargs): + if path == "/proc/mounts": + import io + + return io.StringIO(proc_mounts_contents) + return real_open(path, *args, **kwargs) + + return fake_open + + def test_is_on_hf_mount_linux_match(self): + from transformers.modeling_utils import _is_on_hf_mount + + mounts = ( + "proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0\n" + "hf-mount /data fuse.hf-mount rw,nosuid,nodev,relatime,user_id=0 0 0\n" + ) + with patch("sys.platform", "linux"), patch("builtins.open", self._fake_open_factory(mounts)): + self.assertTrue(_is_on_hf_mount("/data/model.safetensors")) + + def test_is_on_hf_mount_no_match(self): + from transformers.modeling_utils import _is_on_hf_mount + + mounts = "proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0\n/dev/nvme0n1p1 /data ext4 rw,relatime 0 0\n" + with patch("sys.platform", "linux"), patch("builtins.open", self._fake_open_factory(mounts)): + self.assertFalse(_is_on_hf_mount("/data/model.safetensors")) + + def test_is_on_hf_mount_non_linux(self): + from transformers.modeling_utils import _is_on_hf_mount + + with patch("sys.platform", "darwin"): + self.assertFalse(_is_on_hf_mount("/data/model.safetensors")) + + def test_load_state_dict_disable_mmap_explicit(self): + import torch + from safetensors.torch import save_file as safe_save_file + + from transformers.modeling_utils import load_state_dict + + state_dict = { + "weight": torch.arange(12, dtype=torch.float32).reshape(3, 4), + "bias": torch.tensor([1.0, 2.0, 3.0]), + } + with tempfile.TemporaryDirectory() as tmpdir: + ckpt_path = os.path.join(tmpdir, "model.safetensors") + safe_save_file(state_dict, ckpt_path) + + loaded_mmap = load_state_dict(ckpt_path, disable_mmap=False) + loaded_no_mmap = load_state_dict(ckpt_path, disable_mmap=True) + + self.assertEqual(set(loaded_mmap.keys()), set(loaded_no_mmap.keys())) + for k in loaded_mmap: + torch.testing.assert_close(loaded_mmap[k], loaded_no_mmap[k]) diff --git a/utils/check_config_attributes.py b/utils/check_config_attributes.py index b0496b38a10f..65a202cd3f9a 100644 --- a/utils/check_config_attributes.py +++ b/utils/check_config_attributes.py @@ -84,8 +84,6 @@ "AutoformerConfig": ["num_static_real_features", "num_time_features"], "SamVisionConfig": ["mlp_ratio"], "Sam3VisionConfig": ["backbone_feature_sizes"], - "Sam3LiteTextViTConfig": ["global_attn_indexes", "window_size"], - "Sam3LiteTextVisionConfig": ["fpn_hidden_size", "scale_factors"], "SamHQVisionConfig": ["mlp_ratio"], "ClapAudioConfig": ["num_classes"], "ClvpDecoderConfig": ["add_cross_attention"], diff --git a/utils/check_modular_conversion.py b/utils/check_modular_conversion.py index 9fb7d6e7ff56..c6bb10dd2905 100644 --- a/utils/check_modular_conversion.py +++ b/utils/check_modular_conversion.py @@ -106,6 +106,23 @@ def compare_files(modular_file_path, show_diff=True): return diff +# Changes to any of these files can alter the generated output for every modular model, +# so touching them must force a full re-check (see `converter_changed_in_diff`). +CONVERTER_FILES = { + "utils/modular_model_converter.py", + "utils/create_dependency_mapping.py", +} + + +def _get_modified_files(): + fork_point_sha = subprocess.check_output("git merge-base main HEAD".split()).decode("utf-8") + return ( + subprocess.check_output(f"git diff --diff-filter=d --name-only {fork_point_sha}".split()) + .decode("utf-8") + .split() + ) + + def get_models_in_diff(): """ Finds all models that have been modified in the diff. @@ -113,12 +130,7 @@ def get_models_in_diff(): Returns: A set containing the names of the models that have been modified (e.g. {'llama', 'whisper'}). """ - fork_point_sha = subprocess.check_output("git merge-base main HEAD".split()).decode("utf-8") - modified_files = ( - subprocess.check_output(f"git diff --diff-filter=d --name-only {fork_point_sha}".split()) - .decode("utf-8") - .split() - ) + modified_files = _get_modified_files() # Matches both modelling files and tests relevant_modified_files = [x for x in modified_files if "/models/" in x and x.endswith(".py")] @@ -129,6 +141,11 @@ def get_models_in_diff(): return model_names +def converter_changed_in_diff(): + """Whether the diff touches a file that can change conversion output for every model.""" + return any(f in CONVERTER_FILES for f in _get_modified_files()) + + def guaranteed_no_diff(modular_file_path, dependencies, models_in_diff): """ Returns whether it is guaranteed to have no differences between the modular file and the modeling file. @@ -187,6 +204,12 @@ def guaranteed_no_diff(modular_file_path, dependencies, models_in_diff): "[bold red]You are developing on the main branch. We cannot identify the list of changed files and will have to check all files. This may take a while.[/bold red]" ) models_in_diff = {file_path.split("/")[-2] for file_path in args.files} + elif converter_changed_in_diff(): + # The converter (or its dependency-mapping helper) is in the diff: its output can shift + # for any model, so restrict-by-diff would miss regressions. Force a full check. + console.print("[bold yellow]Converter change detected in diff; checking all modular files.[/bold yellow]") + args.check_all = True + models_in_diff = {file_path.split("/")[-2] for file_path in args.files} else: models_in_diff = get_models_in_diff() if not models_in_diff and not args.check_all: diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index 649abd5d62b2..018316680ece 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -1303,6 +1303,47 @@ def _code(node: cst.CSTNode) -> str: return other_imports + result +def replace_unprotected_image_processing_imports(files: dict, all_imports: list) -> dict: + """ + Because `image_processing` file uses non-protected torchvision and torch imports, we need to duplicate the nodes + inside `image_processing_pil` instead of importing them directly from `.image_processing_xxx`, which would crash if + torchvision is not installed. + """ + if not ("image_processing" in files and "image_processing_pil" in files): + return files + + body = files["image_processing_pil"] + needed_imports = get_needed_imports(body, all_imports) + import_from_image_processing = None + for import_node in needed_imports: + if isinstance(import_node, cst.SimpleStatementLine) and isinstance(import_node.body[0], cst.ImportFrom): + import_node = import_node.body[0] + full_name = get_full_attribute_name(import_node.module) + if re.search(r"^image_processing_(?!(?:backends)|(?:utils))", full_name): + import_from_image_processing = import_node + break + + if import_from_image_processing is None: + return files + + imported_objects = [x.name.value for x in import_from_image_processing.names] + nodes_to_add = {name: files["image_processing"][name] for name in imported_objects} + # Update the position inside the final file + for name, node_structure in nodes_to_add.items(): + node_with_same_index = next( + v["node"] for v in body.values() if v["insert_idx"] == node_structure["insert_idx"] + ) + # Insert the new node before the corresponding node if the corresponding node is a class or function + if isinstance(node_with_same_index, (cst.ClassDef, cst.FunctionDef)): + nodes_to_add[name]["insert_idx"] -= 0.5 + # Otherwise, after it + else: + nodes_to_add[name]["insert_idx"] += 0.5 + # Add the nodes inside the body of `image_processing_pil` + body.update(nodes_to_add) + return files + + def split_all_assignment(node: cst.CSTNode, model_name: str) -> dict[str, cst.CSTNode]: """Split the `__all__` assignment found in the modular between each corresponding files.""" all_all_per_file = {} @@ -1692,10 +1733,6 @@ class NewNameModel(LlamaModel): class_file_type = find_file_type(class_name, new_name) # In this case, we need to remove it from the dependencies and create a new import instead if class_file_type != file_type: - # image_processing_pil and image_processing must never depend on each other. - # When a PIL class needs an image_processing class, inline it instead of importing. - if file_type == "image_processing_pil" and class_file_type == "image_processing": - continue corrected_dependencies.remove(class_name) import_statement = f"from .{class_file_type}_{new_name} import {class_name}" new_imports[class_name] = cst.parse_statement(import_statement) @@ -1748,14 +1785,7 @@ class node based on the inherited classes if needed. Also returns any new import # Remove all classes explicitly defined in modular from the dependencies. Otherwise, if a class is referenced # before its new modular definition, it may be wrongly imported from elsewhere as a dependency if it matches # another class from a modeling file after renaming, even though it would be added after anyway (leading to duplicates) - # Exception: for image_processing_pil files, image_processing modular classes must be inlined (not excluded), - # because these two files must never import from each other. - classes_to_exclude = set(modular_mapper.classes.keys()) - if file_type == "image_processing_pil": - classes_to_exclude -= { - k for k in classes_to_exclude if find_file_type(k, model_name) == "image_processing" - } - new_node_dependencies -= classes_to_exclude + new_node_dependencies -= set(modular_mapper.classes.keys()) # The node was modified -> look for all recursive dependencies of the new node all_dependencies_to_add = find_all_dependencies( @@ -1856,6 +1886,11 @@ def create_modules( all_imports.extend(new_imports) all_imports_code.update(new_imports_code) + # Because `image_processing` file uses non-protected torchvision and torch imports, we need to duplicate the nodes + # here instead of importing from `.image_processing_model`, which would crash if torchvision is not installed + if "image_processing" in files and "image_processing_pil" in files: + files = replace_unprotected_image_processing_imports(files, all_imports) + # Find the correct imports, and write the new modules for file, body in files.items(): new_body = [k[1]["node"] for k in sorted(body.items(), key=lambda x: x[1]["insert_idx"])]