From 028e50262806e30e9c1c4517cbac3e1f9208e190 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 25 Jul 2024 16:10:48 +0200 Subject: [PATCH 01/68] first try --- src/transformers/modeling_utils.py | 37 +++++++++++++++++++ .../models/blip_2/modeling_blip_2.py | 12 ++++-- .../models/idefics2/modeling_idefics2.py | 17 +++++---- .../instructblip/modeling_instructblip.py | 8 ++-- .../modeling_instructblipvideo.py | 8 ++-- .../models/llava/modeling_llava.py | 15 ++------ .../models/llava_next/modeling_llava_next.py | 6 ++- .../modeling_llava_next_video.py | 5 +-- .../models/paligemma/modeling_paligemma.py | 6 ++- .../video_llava/modeling_video_llava.py | 10 +++-- .../models/vipllava/modeling_vipllava.py | 6 ++- tests/models/blip/test_modeling_blip.py | 24 ++++++++++++ tests/models/blip_2/test_modeling_blip_2.py | 12 ++++++ .../models/idefics2/test_modeling_idefics2.py | 12 ++++++ .../test_modeling_instructblip.py | 6 +++ .../test_modeling_instructblipvideo.py | 6 +++ tests/models/kosmos2/test_modeling_kosmos2.py | 6 +++ tests/models/llava/test_modeling_llava.py | 5 +++ .../llava_next/test_modeling_llava_next.py | 5 +++ .../test_modeling_llava_next_video.py | 5 +++ .../paligemma/test_modeling_paligemma.py | 6 +++ .../video_llava/test_modeling_video_llava.py | 5 +++ .../models/vipllava/test_modeling_vipllava.py | 5 +++ tests/test_modeling_common.py | 19 ++++++++-- 24 files changed, 201 insertions(+), 45 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 8f1ad56f6999..7f5da22eb15b 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -45,6 +45,7 @@ from .dynamic_module_utils import custom_object_save from .generation import GenerationConfig, GenerationMixin from .integrations import PeftAdapterMixin, deepspeed_config, is_deepspeed_zero3_enabled +from .models.auto.modeling_auto import MODEL_MAPPING from .pytorch_utils import ( # noqa: F401 Conv1D, apply_chunking_to_forward, @@ -1512,6 +1513,42 @@ def _autoset_attn_implementation( # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available. requested_attn_implementation = config._attn_implementation_internal + # MultiModal LLM related hack: since they consist of two or might be more sub-models + # we have to check and dispatch SDPA to each sub-model, in case any of them support it. + # If one sub-model supports SDPA while other doesn't, an error will be raised following the + # typical SDPA-dispatch path. + # Same goes for the `_supports_cache_class` because we cannot know what flag LM has + # before knowing which class is that LM + if hasattr(config, "text_config") and hasattr(config, "vision_config"): + text_model_cls = MODEL_MAPPING.get(type(config.text_config), None) + if text_model_cls is not None: + cls._supports_cache_class = text_model_cls._supports_cache_class + + config.text_config._attn_implementation = config._attn_implementation + config.vision_config._attn_implementation = config._attn_implementation + cls._supports_sdpa = text_model_cls._supports_sdpa + cls._autoset_attn_implementation( + config.text_config, + use_flash_attention_2=use_flash_attention_2, + torch_dtype=torch_dtype, + device_map=device_map, + check_device_map=check_device_map, + ) + + vision_model_cls = MODEL_MAPPING.get(type(config.vision_config), None) + if vision_model_cls is not None: + cls._supports_sdpa = vision_model_cls._supports_sdpa + cls._autoset_attn_implementation( + config.vision_config, + use_flash_attention_2=use_flash_attention_2, + torch_dtype=torch_dtype, + device_map=device_map, + check_device_map=check_device_map, + ) + + if vision_model_cls is not None and text_model_cls is not None: + cls._supports_sdpa = vision_model_cls._supports_sdpa or text_model_cls._supports_sdpa + if use_flash_attention_2: logger.warning_once( 'The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation="flash_attention_2"` instead.' diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 7aad5bea66ca..2dcf8a6ad625 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1225,7 +1225,9 @@ class Blip2Model(Blip2PreTrainedModel): def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel(config.vision_config) + self.vision_model = Blip2VisionModel( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = Blip2QFormerModel(config.qformer_config) @@ -1233,11 +1235,11 @@ def __init__(self, config: Blip2Config): self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) # Update _tied_weights_keys using the base model used. @@ -1590,7 +1592,9 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel): def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel(config.vision_config) + self.vision_model = Blip2VisionModel( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = Blip2QFormerModel(config.qformer_config) diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index f57f1fc3d51a..34ba8201b8aa 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -1123,14 +1123,15 @@ def _autoset_attn_implementation( """ Overrides the method in `PreTrainedModel` to update the vision config with the correct attention implementation """ - config = super()._autoset_attn_implementation( - config=config, - use_flash_attention_2=use_flash_attention_2, - torch_dtype=torch_dtype, - device_map=device_map, - check_device_map=check_device_map, - **kwargs, - ) + config._attn_implementation = "eager" + # = super()._autoset_attn_implementation( + # config=config, + # use_flash_attention_2=use_flash_attention_2, + # torch_dtype=torch_dtype, + # device_map=device_map, + # check_device_map=check_device_map, + # **kwargs, + # ) config.vision_config._attn_implementation = config._attn_implementation return config diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 8ad47b308fd0..da2cb7fbbc9c 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1281,7 +1281,9 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel): def __init__(self, config: InstructBlipConfig): super().__init__(config) - self.vision_model = InstructBlipVisionModel(config.vision_config) + self.vision_model = InstructBlipVisionModel( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = InstructBlipQFormerModel(config.qformer_config) @@ -1290,11 +1292,11 @@ def __init__(self, config: InstructBlipConfig): if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) if language_model._no_split_modules is not None: diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index d3b594e9c3f7..7ed9e0fe5440 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1290,7 +1290,9 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel def __init__(self, config: InstructBlipVideoConfig): super().__init__(config) - self.vision_model = InstructBlipVideoVisionModel(config.vision_config) + self.vision_model = InstructBlipVideoVisionModel( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = InstructBlipVideoQFormerModel(config.qformer_config) @@ -1299,11 +1301,11 @@ def __init__(self, config: InstructBlipVideoConfig): if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) if language_model._no_split_modules is not None: diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 0426776beed1..b5309f0f9d4a 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -125,7 +125,6 @@ class LlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True def _init_weights(self, module): # important: this ported version of Llava isn't meant for training from scratch - only @@ -149,14 +148,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @property - def _supports_sdpa(self): - """ - Retrieve language_model's attribute to check whether the model supports - SDPA or not. - """ - return self.language_model._supports_sdpa - LLAVA_INPUTS_DOCSTRING = r""" Args: @@ -236,12 +227,14 @@ def _supports_sdpa(self): class LlavaForConditionalGeneration(LlavaPreTrainedModel): def __init__(self, config: LlavaConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config(config.vision_config) + self.vision_tower = AutoModel.from_config( + config.vision_config, attn_implementation=config.text_config._attn_implementation + ) self.multi_modal_projector = LlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.vision_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index ad76561df54f..38e8954aca25 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -345,7 +345,9 @@ def _supports_sdpa(self): class LlavaNextForConditionalGeneration(LlavaNextPreTrainedModel): def __init__(self, config: LlavaNextConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config(config.vision_config) + self.vision_tower = AutoModel.from_config( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.multi_modal_projector = LlavaNextMultiModalProjector(config) embed_std = 1 / math.sqrt(config.text_config.hidden_size) @@ -353,7 +355,7 @@ def __init__(self, config: LlavaNextConfig): self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index e3264dfd91e1..e4b3a6ce1a7e 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -388,15 +388,14 @@ def __init__( config: LlavaNextVideoConfig, ): super().__init__(config) - self.vision_tower = AutoModel.from_config(config.vision_config) + self.vision_tower = AutoModel.from_config(config.vision_config, attn_implementation=config.vision_config._attn_implementation) - self.multi_modal_projector = LlavaNextVideoMultiModalProjector(config) embed_std = 1 / math.sqrt(config.text_config.hidden_size) self.image_newline = nn.Parameter(torch.randn(config.text_config.hidden_size, dtype=self.dtype) * embed_std) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 8a693e56f80b..16bcf2475125 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -231,13 +231,15 @@ def _supports_sdpa(self): class PaliGemmaForConditionalGeneration(PaliGemmaPreTrainedModel): def __init__(self, config: PaliGemmaConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config(config=config.vision_config) + self.vision_tower = AutoModel.from_config( + config=config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.multi_modal_projector = PaliGemmaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self._attn_implementation = config._attn_implementation language_model = AutoModelForCausalLM.from_config( - config=config.text_config, attn_implementation=self._attn_implementation + config=config.text_config, attn_implementation=self.text_config._attn_implementation ) if language_model._tied_weights_keys is not None: diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index cb54c433fde8..aea575738316 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -237,13 +237,17 @@ def _supports_sdpa(self): class VideoLlavaForConditionalGeneration(VideoLlavaPreTrainedModel): def __init__(self, config: VideoLlavaConfig): super().__init__(config) - self.video_tower = AutoModel.from_config(config.vision_config) - self.image_tower = AutoModel.from_config(config.vision_config) + self.video_tower = AutoModel.from_config( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) + self.image_tower = AutoModel.from_config( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.multi_modal_projector = VideoLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index c5f856e78745..1483461beeef 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -241,12 +241,14 @@ def _supports_sdpa(self): class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel): def __init__(self, config: VipLlavaConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config(config.vision_config) + self.vision_tower = AutoModel.from_config( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.multi_modal_projector = VipLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 2f8ee3229ff2..981f4523c7e9 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -446,6 +446,12 @@ class BlipModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = BlipModelTester(self) @@ -805,6 +811,12 @@ class BlipVQAModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = BlipVQAModelTester(self) @@ -885,6 +897,12 @@ class BlipTextRetrievalModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = BlipTextRetrievalModelTester(self) @@ -1113,6 +1131,12 @@ class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = BlipTextImageModelsModelTester(self) diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 28ed3a79cae5..b13110268cc9 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -445,6 +445,12 @@ class Blip2ForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, GenerationT test_attention_outputs = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = Blip2ForConditionalGenerationDecoderOnlyModelTester(self) @@ -704,6 +710,12 @@ class Blip2ModelTest(ModelTesterMixin, PipelineTesterMixin, GenerationTesterMixi test_attention_outputs = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = Blip2ModelTester(self) diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 057ce93cd87e..8386021a7ebb 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -172,6 +172,12 @@ class Idefics2ModelTest(ModelTesterMixin, unittest.TestCase): test_resize_embeddings = True test_head_masking = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because LM/vision models used in tests dont support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = Idefics2VisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=Idefics2Config, has_text_modality=False) @@ -332,6 +338,12 @@ class Idefics2ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTest test_head_masking = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because LM/vision models used in tests dont support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = Idefics2VisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=Idefics2Config, has_text_modality=False) diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 1aaa8e1a8b68..939bbb3d001a 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -461,6 +461,12 @@ class InstructBlipForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, Gene test_attention_outputs = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = InstructBlipForConditionalGenerationDecoderOnlyModelTester(self) diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 1265db3a2a2e..c1d0d5113c54 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -482,6 +482,12 @@ class InstructBlipVideoForConditionalGenerationDecoderOnlyTest( test_attention_outputs = False test_torchscript = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because vision models used in tests don't support SDPA + supports_sdpa = False + def setUp(self): self.model_tester = InstructBlipVideoForConditionalGenerationDecoderOnlyModelTester(self) diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index 6f34689004ef..c7aa4d9aca48 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -258,6 +258,12 @@ class Kosmos2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_resize_embeddings = False test_attention_outputs = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to False because LM/vision models used in tests don't support SDPA + supports_sdpa = False + # TODO: `image-to-text` pipeline for this model needs Processor. def is_pipeline_test_to_skip( self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name diff --git a/tests/models/llava/test_modeling_llava.py b/tests/models/llava/test_modeling_llava.py index b37e4df3cc10..5e9e336f554e 100644 --- a/tests/models/llava/test_modeling_llava.py +++ b/tests/models/llava/test_modeling_llava.py @@ -181,6 +181,11 @@ class LlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase pipeline_model_mapping = {"image-to-text": LlavaForConditionalGeneration} if is_torch_available() else {} test_pruning = False test_head_masking = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA + supports_sdpa = True def setUp(self): self.model_tester = LlavaVisionText2TextModelTester(self) diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index 70d91002a91b..3234f001f951 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -216,6 +216,11 @@ class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, GenerationTes all_model_classes = (LlavaNextForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA + supports_sdpa = True def setUp(self): self.model_tester = LlavaNextVisionText2TextModelTester(self) diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 9ba7ef869ddf..d22e8a1dc824 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -231,6 +231,11 @@ class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, Generati all_model_classes = (LlavaNextVideoForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA + supports_sdpa = True def setUp(self): self.model_tester = LlavaNextVideoVisionText2TextModelTester(self) diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index 7753ae073dd3..ce1d5eb5686a 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -181,6 +181,12 @@ class PaliGemmaForConditionalGenerationModelTest(ModelTesterMixin, unittest.Test test_torchscript = False test_head_masking = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA + supports_sdpa = True + def setUp(self): self.model_tester = PaliGemmaVisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=PaliGemmaConfig, has_text_modality=False) diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index fe3eea97dcf3..10aeac71256e 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -200,6 +200,11 @@ class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTe test_pruning = False test_resize_embeddings = True test_head_masking = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA + supports_sdpa = True def setUp(self): self.model_tester = VideoLlavaVisionText2TextModelTester(self) diff --git a/tests/models/vipllava/test_modeling_vipllava.py b/tests/models/vipllava/test_modeling_vipllava.py index a4e89d3f9ddf..51cf66d856f7 100644 --- a/tests/models/vipllava/test_modeling_vipllava.py +++ b/tests/models/vipllava/test_modeling_vipllava.py @@ -162,6 +162,11 @@ class VipLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestC test_pruning = False test_resize_embeddings = True test_head_masking = False + is_multimodal = True + # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # So we can't know if SDPA is supported before starting to load the model + # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA + supports_sdpa = True def setUp(self): self.model_tester = VipLlavaVisionText2TextModelTester(self) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index abe5ddea2c25..cc4abaf88e57 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -184,6 +184,7 @@ class ModelTesterMixin: is_encoder_decoder = False has_attentions = True model_split_percents = [0.5, 0.7, 0.9] + is_multimodal = False def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = copy.deepcopy(inputs_dict) @@ -3711,7 +3712,9 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa: + if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( + self.is_multimodal and not self.supports_sdpa + ): self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -4019,7 +4022,9 @@ def test_sdpa_can_dispatch_on_flash(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if not model_class._supports_sdpa: + if (not self.is_multimodal and not model_class._supports_sdpa) or ( + self.is_multimodal and not self.supports_sdpa + ): self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4065,7 +4070,9 @@ def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if not model_class._supports_sdpa: + if (not self.is_multimodal and not model_class._supports_sdpa) or ( + self.is_multimodal and not self.supports_sdpa + ): self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4107,7 +4114,9 @@ def test_eager_matches_sdpa_generate(self): self.skipTest(f"{self.__class__.__name__} tests a model that does support generate: skipping this test") for model_class in self.all_generative_model_classes: - if not model_class._supports_sdpa: + if (not self.is_multimodal and not model_class._supports_sdpa) or ( + self.is_multimodal and not self.supports_sdpa + ): self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4180,6 +4189,8 @@ def test_sdpa_matches_eager_sliding_window(self): self.skipTest(f"No generative model classes for {self.__class__.__name__}") for model_class in self.all_generative_model_classes: + if model_class._supports_sdpa: + self.skipTest(reason="Model architecture does not support attentions") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() if config.model_type not in WINDOW_ATTENTION_MODELS: From 589d18af14ffb1c4cf65b399d111be9785e07942 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 25 Jul 2024 16:11:02 +0200 Subject: [PATCH 02/68] codestyle --- .../models/llava_next_video/modeling_llava_next_video.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index e4b3a6ce1a7e..ecab496b20bc 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -388,7 +388,9 @@ def __init__( config: LlavaNextVideoConfig, ): super().__init__(config) - self.vision_tower = AutoModel.from_config(config.vision_config, attn_implementation=config.vision_config._attn_implementation) + self.vision_tower = AutoModel.from_config( + config.vision_config, attn_implementation=config.vision_config._attn_implementation + ) embed_std = 1 / math.sqrt(config.text_config.hidden_size) self.image_newline = nn.Parameter(torch.randn(config.text_config.hidden_size, dtype=self.dtype) * embed_std) From b33982ff19b8ce3a36f70a5336b7e937c8da2cf2 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 08:31:43 +0200 Subject: [PATCH 03/68] idefics2 is happy --- .../models/idefics2/modeling_idefics2.py | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 34ba8201b8aa..e57be17f5096 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -921,7 +921,9 @@ def __init__(self, config, layer_idx: int): self.input_latents_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.input_context_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) - self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx) + self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[config.perceiver_config._attn_implementation]( + config, layer_idx=layer_idx + ) self.post_attention_layernorm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.mlp = Idefics2MLP( hidden_size=config.text_config.hidden_size, @@ -1123,16 +1125,26 @@ def _autoset_attn_implementation( """ Overrides the method in `PreTrainedModel` to update the vision config with the correct attention implementation """ - config._attn_implementation = "eager" - # = super()._autoset_attn_implementation( - # config=config, - # use_flash_attention_2=use_flash_attention_2, - # torch_dtype=torch_dtype, - # device_map=device_map, - # check_device_map=check_device_map, - # **kwargs, - # ) - config.vision_config._attn_implementation = config._attn_implementation + config = super()._autoset_attn_implementation( + config=config, + use_flash_attention_2=use_flash_attention_2, + torch_dtype=torch_dtype, + device_map=device_map, + check_device_map=check_device_map, + **kwargs, + ) + # autoset-attn calls recursively all sub-configs (text-config, vision-config) + # and sets attn implementation if the config can be mapped bu auto-model + # Idefics2 vision config can't be mapped automcatically so we set it manually here + # We cant set vision attn same as general attn, because the general one can be sdpa if at + # least one sub-module (in this case LLM) supports sdpa + if hasattr(config, "vision_config"): + config.vision_config._attn_implementation = ( + config._attn_implementation if config._attn_implementation != "sdpa" else "eager" + ) + config.perceiver_config._attn_implementation = ( + config._attn_implementation if config._attn_implementation != "sdpa" else "eager" + ) return config From b0baa75ed4cb90233e50c99943286e1e3fc6039c Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 08:45:36 +0200 Subject: [PATCH 04/68] [run-slow] llava, llava_next, video_llava, vipllava, llava_next_video, idefics, idefics2, kosmos2, fuyu, blip, blip_2, instructblip, instructblipvideo, paligemma --- src/transformers/modeling_utils.py | 2 -- .../models/paligemma/modeling_paligemma.py | 11 +---------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 7f5da22eb15b..6fe8d76e629e 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1522,8 +1522,6 @@ def _autoset_attn_implementation( if hasattr(config, "text_config") and hasattr(config, "vision_config"): text_model_cls = MODEL_MAPPING.get(type(config.text_config), None) if text_model_cls is not None: - cls._supports_cache_class = text_model_cls._supports_cache_class - config.text_config._attn_implementation = config._attn_implementation config.vision_config._attn_implementation = config._attn_implementation cls._supports_sdpa = text_model_cls._supports_sdpa diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 16bcf2475125..2fffb36f1cd3 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -126,7 +126,6 @@ class PaliGemmaPreTrainedModel(PreTrainedModel): _no_split_modules = ["PaliGemmaMultiModalProjector"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = False - _supports_sdpa = True def _init_weights(self, module): # important: this ported version of PaliGemmaisn't meant for training from scratch - only @@ -149,14 +148,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @property - def _supports_sdpa(self): - """ - Retrieve language_model's attribute to check whether the model supports - SDPA or not. - """ - return self.language_model._supports_sdpa - PALIGEMMA_INPUTS_DOCSTRING = r""" Args: @@ -239,7 +230,7 @@ def __init__(self, config: PaliGemmaConfig): self._attn_implementation = config._attn_implementation language_model = AutoModelForCausalLM.from_config( - config=config.text_config, attn_implementation=self.text_config._attn_implementation + config=config.text_config, attn_implementation=config.text_config._attn_implementation ) if language_model._tied_weights_keys is not None: From 9f19211300dcb922fc314bdb6855d14124f4c8eb Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 08:59:12 +0200 Subject: [PATCH 05/68] fix-copies --- src/transformers/models/llava/modeling_llava.py | 1 + .../models/llava_next/modeling_llava_next.py | 8 -------- .../llava_next_video/modeling_llava_next_video.py | 8 -------- .../models/vipllava/modeling_vipllava.py | 12 ++---------- tests/models/musicgen/test_modeling_musicgen.py | 8 ++++++-- .../musicgen_melody/test_modeling_musicgen_melody.py | 4 +++- 6 files changed, 12 insertions(+), 29 deletions(-) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index b5309f0f9d4a..c1c373565c48 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -125,6 +125,7 @@ class LlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True def _init_weights(self, module): # important: this ported version of Llava isn't meant for training from scratch - only diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 38e8954aca25..0687c4c1d534 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -255,14 +255,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @property - def _supports_sdpa(self): - """ - Retrieve language_model's attribute to check whether the model supports - SDPA or not. - """ - return self.language_model._supports_sdpa - LLAVA_NEXT_INPUTS_DOCSTRING = r""" Args: diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index ecab496b20bc..92d9288560dd 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -295,14 +295,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @property - def _supports_sdpa(self): - """ - Retrieve language_model's attribute to check whether the model supports - SDPA or not. - """ - return self.language_model._supports_sdpa - LLAVA_NEXT_VIDEO_INPUTS_DOCSTRING = r""" Args: diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 1483461beeef..310bc9d8d9e6 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -158,14 +158,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @property - def _supports_sdpa(self): - """ - Retrieve language_model's attribute to check whether the model supports - SDPA or not. - """ - return self.language_model._supports_sdpa - VIPLLAVA_INPUTS_DOCSTRING = r""" Args: @@ -242,13 +234,13 @@ class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel): def __init__(self, config: VipLlavaConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config.text_config._attn_implementation ) self.multi_modal_projector = VipLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config.vision_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 7fc2f8c9db47..6ad93d3268ec 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -627,7 +627,9 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa: + if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( + self.is_multimodal and not self.supports_sdpa + ): self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -1942,7 +1944,9 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa: + if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( + self.is_multimodal and not self.supports_sdpa + ): self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index 7cebf037d27a..f8486f5653c7 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -629,7 +629,9 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa: + if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( + self.is_multimodal and not self.supports_sdpa + ): self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): From 19e0f3fc64a9d2632dfeb56304b065e77afe7a02 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 08:59:31 +0200 Subject: [PATCH 06/68] [run-slow] llava, llava_next, video_llava, vipllava, llava_next_video, idefics, idefics2, kosmos2, fuyu, blip, blip_2, instructblip, instructblipvideo From 56a6f81916c511584ed67caa2b12f4ff62101e6b Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 09:34:24 +0200 Subject: [PATCH 07/68] blip-2 needs to init vision from config --- src/transformers/models/blip_2/modeling_blip_2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 2dcf8a6ad625..8028cd1e8777 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1225,7 +1225,7 @@ class Blip2Model(Blip2PreTrainedModel): def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel( + self.vision_model = Blip2VisionModel._from_config( config.vision_config, attn_implementation=config.vision_config._attn_implementation ) @@ -1592,7 +1592,7 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel): def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel( + self.vision_model = Blip2VisionModel._from_config( config.vision_config, attn_implementation=config.vision_config._attn_implementation ) From bbff1ac04650d252e3f8e808f8df5b043e8143fb Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 11:23:27 +0200 Subject: [PATCH 08/68] when was this removed O_o --- .../models/llava_next_video/modeling_llava_next_video.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 92d9288560dd..11247f037069 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -383,6 +383,7 @@ def __init__( self.vision_tower = AutoModel.from_config( config.vision_config, attn_implementation=config.vision_config._attn_implementation ) + self.multi_modal_projector = LlavaNextVideoMultiModalProjector(config) embed_std = 1 / math.sqrt(config.text_config.hidden_size) self.image_newline = nn.Parameter(torch.randn(config.text_config.hidden_size, dtype=self.dtype) * embed_std) From 8485df9aaa09445b8a1f2d706df0e95bd1370821 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 11:24:53 +0200 Subject: [PATCH 09/68] minor fix --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6fe8d76e629e..911dc5f24391 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1523,7 +1523,6 @@ def _autoset_attn_implementation( text_model_cls = MODEL_MAPPING.get(type(config.text_config), None) if text_model_cls is not None: config.text_config._attn_implementation = config._attn_implementation - config.vision_config._attn_implementation = config._attn_implementation cls._supports_sdpa = text_model_cls._supports_sdpa cls._autoset_attn_implementation( config.text_config, @@ -1535,6 +1534,7 @@ def _autoset_attn_implementation( vision_model_cls = MODEL_MAPPING.get(type(config.vision_config), None) if vision_model_cls is not None: + config.vision_config._attn_implementation = config._attn_implementation cls._supports_sdpa = vision_model_cls._supports_sdpa cls._autoset_attn_implementation( config.vision_config, From ba7ee7f6aa50bfcc1886ec97ea34e18eb56f3e0f Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 26 Jul 2024 11:44:07 +0200 Subject: [PATCH 10/68] tests --- .../models/instructblipvideo/modeling_instructblipvideo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 7ed9e0fe5440..62330de6042d 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1290,7 +1290,7 @@ class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel def __init__(self, config: InstructBlipVideoConfig): super().__init__(config) - self.vision_model = InstructBlipVideoVisionModel( + self.vision_model = InstructBlipVideoVisionModel._from_config( config.vision_config, attn_implementation=config.vision_config._attn_implementation ) From d793d04b8e67f8ce9953bd9fdb14a474ec442014 Mon Sep 17 00:00:00 2001 From: raushan Date: Mon, 29 Jul 2024 15:46:13 +0200 Subject: [PATCH 11/68] this way? --- src/transformers/modeling_utils.py | 27 +++++++++++-------- .../models/llava/modeling_llava.py | 4 +-- .../models/vipllava/modeling_vipllava.py | 4 +-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 911dc5f24391..6c267ab91f64 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1517,14 +1517,15 @@ def _autoset_attn_implementation( # we have to check and dispatch SDPA to each sub-model, in case any of them support it. # If one sub-model supports SDPA while other doesn't, an error will be raised following the # typical SDPA-dispatch path. - # Same goes for the `_supports_cache_class` because we cannot know what flag LM has - # before knowing which class is that LM if hasattr(config, "text_config") and hasattr(config, "vision_config"): + # set to None to avoid hard_check errors, because general VLM's `_support_sdpa` attr is always `False` by default + requested_attn_implementation = ( + None if requested_attn_implementation == "sdpa" else requested_attn_implementation + ) text_model_cls = MODEL_MAPPING.get(type(config.text_config), None) if text_model_cls is not None: - config.text_config._attn_implementation = config._attn_implementation - cls._supports_sdpa = text_model_cls._supports_sdpa - cls._autoset_attn_implementation( + config.text_config._attn_implementation = requested_attn_implementation + text_model_cls._autoset_attn_implementation( config.text_config, use_flash_attention_2=use_flash_attention_2, torch_dtype=torch_dtype, @@ -1534,9 +1535,8 @@ def _autoset_attn_implementation( vision_model_cls = MODEL_MAPPING.get(type(config.vision_config), None) if vision_model_cls is not None: - config.vision_config._attn_implementation = config._attn_implementation - cls._supports_sdpa = vision_model_cls._supports_sdpa - cls._autoset_attn_implementation( + config.vision_config._attn_implementation = requested_attn_implementation + vision_model_cls._autoset_attn_implementation( config.vision_config, use_flash_attention_2=use_flash_attention_2, torch_dtype=torch_dtype, @@ -1544,9 +1544,6 @@ def _autoset_attn_implementation( check_device_map=check_device_map, ) - if vision_model_cls is not None and text_model_cls is not None: - cls._supports_sdpa = vision_model_cls._supports_sdpa or text_model_cls._supports_sdpa - if use_flash_attention_2: logger.warning_once( 'The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation="flash_attention_2"` instead.' @@ -1739,6 +1736,14 @@ def _check_and_enable_sdpa(cls, config, hard_check_only: bool = False) -> Pretra "PyTorch SDPA requirements in Transformers are not met. Please install torch>=2.1.1." ) + # VLMs have to follow the sdpa attr of its sub-configs + if hasattr(config, "text_config") and hasattr(config, "vision_config"): + if ( + config.text_config._attn_implementation == "sdpa" + or config.vision_config._attn_implementation == "sdpa" + ): + config._attn_implementation = "sdpa" + if not is_torch_sdpa_available() or not cls._supports_sdpa: return config diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index c1c373565c48..9c9bcf95dcb2 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -229,13 +229,13 @@ class LlavaForConditionalGeneration(LlavaPreTrainedModel): def __init__(self, config: LlavaConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.text_config._attn_implementation + config.vision_config, attn_implementation=config.vision_config._attn_implementation ) self.multi_modal_projector = LlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.vision_config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 310bc9d8d9e6..58ebf2db2dea 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -234,13 +234,13 @@ class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel): def __init__(self, config: VipLlavaConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.text_config._attn_implementation + config.vision_config, attn_implementation=config.vision_config._attn_implementation ) self.multi_modal_projector = VipLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.vision_config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() From 58aff2757811710a1c835f2450d764cb030d8071 Mon Sep 17 00:00:00 2001 From: raushan Date: Mon, 29 Jul 2024 15:49:53 +0200 Subject: [PATCH 12/68] tests --- src/transformers/models/instructblip/modeling_instructblip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index da2cb7fbbc9c..40135148ed3b 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -1281,7 +1281,7 @@ class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel): def __init__(self, config: InstructBlipConfig): super().__init__(config) - self.vision_model = InstructBlipVisionModel( + self.vision_model = InstructBlipVisionModel._from_config( config.vision_config, attn_implementation=config.vision_config._attn_implementation ) From 4e4bd2743aa8b97ff664138b48b27927d098a2e8 Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 09:37:37 +0200 Subject: [PATCH 13/68] model-agnostic code --- src/transformers/configuration_utils.py | 4 + src/transformers/modeling_utils.py | 72 ++-- .../modeling_vision_encoder_decoder.py | 6 +- .../test_modeling_vision_encoder_decoder.py | 342 +++++++++++++++++- 4 files changed, 386 insertions(+), 38 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index 2f84bc29aee2..0a5440c14022 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -831,6 +831,10 @@ def __eq__(self, other): def __repr__(self): return f"{self.__class__.__name__} {self.to_json_string()}" + def __iter__(self): + for attr, value in copy.deepcopy(self.__dict__).items(): + yield attr, value + def to_diff_dict(self) -> Dict[str, Any]: """ Removes all attributes from config which correspond to the default config attributes for better readability and diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6c267ab91f64..f3adfa16c10a 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1513,36 +1513,29 @@ def _autoset_attn_implementation( # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available. requested_attn_implementation = config._attn_implementation_internal - # MultiModal LLM related hack: since they consist of two or might be more sub-models - # we have to check and dispatch SDPA to each sub-model, in case any of them support it. + # MultiModal-LLM/Encoder-Decoder related hack: since they consist of two or might be more sub-configs + # we have to check and dispatch SDPA to each sub-config, in case any of them support it. # If one sub-model supports SDPA while other doesn't, an error will be raised following the - # typical SDPA-dispatch path. - if hasattr(config, "text_config") and hasattr(config, "vision_config"): - # set to None to avoid hard_check errors, because general VLM's `_support_sdpa` attr is always `False` by default + # typical SDPA-dispatch path (i.e. if hard_check) + sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} + if sub_configs: + # Set to None to avoid hard_check errors, because generalist model's `_support_sdpa` attr is usually `False` by default + # while sub-configs can still be set to `True` requested_attn_implementation = ( None if requested_attn_implementation == "sdpa" else requested_attn_implementation ) - text_model_cls = MODEL_MAPPING.get(type(config.text_config), None) - if text_model_cls is not None: - config.text_config._attn_implementation = requested_attn_implementation - text_model_cls._autoset_attn_implementation( - config.text_config, - use_flash_attention_2=use_flash_attention_2, - torch_dtype=torch_dtype, - device_map=device_map, - check_device_map=check_device_map, - ) - - vision_model_cls = MODEL_MAPPING.get(type(config.vision_config), None) - if vision_model_cls is not None: - config.vision_config._attn_implementation = requested_attn_implementation - vision_model_cls._autoset_attn_implementation( - config.vision_config, - use_flash_attention_2=use_flash_attention_2, - torch_dtype=torch_dtype, - device_map=device_map, - check_device_map=check_device_map, - ) + for key, sub_config in sub_configs.items(): + sub_model_cls = MODEL_MAPPING.get(type(sub_config), None) + if sub_model_cls is not None: + sub_config._attn_implementation = requested_attn_implementation + sub_model_cls._autoset_attn_implementation( + sub_config, + use_flash_attention_2=use_flash_attention_2, + torch_dtype=torch_dtype, + device_map=device_map, + check_device_map=check_device_map, + ) + setattr(config, key, sub_config) if use_flash_attention_2: logger.warning_once( @@ -1641,7 +1634,10 @@ def _check_and_enable_flash_attn_2( If all checks pass and `hard_check_only` is False, the method will set the config attribute `attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module. """ - if not cls._supports_flash_attn_2: + + # VLM/Encoder-Decoder etc. have to follow the sdpa attr of its sub-configs + sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} + if not cls._supports_flash_attn_2 and not sub_configs: raise ValueError( f"{cls.__name__} does not support Flash Attention 2.0 yet. Please request to add support where" f" the model is hosted, on its model hub page: https://huggingface.co/{config._name_or_path}/discussions/new" @@ -1714,7 +1710,15 @@ def _check_and_enable_flash_attn_2( "initialise the model on a GPU by passing a device_map that contains only GPU devices as keys." ) if not hard_check_only: - config._attn_implementation = "flash_attention_2" + if sub_configs: + sub_config_attentions = {sub_config._attn_implementation for key, sub_config in sub_configs.items()} + config._attn_implementation = ( + "flash_attention_2" + if "flash_attention_2" in sub_config_attentions + else config._attn_implementation + ) + else: + config._attn_implementation = "flash_attention_2" return config @classmethod @@ -1736,13 +1740,11 @@ def _check_and_enable_sdpa(cls, config, hard_check_only: bool = False) -> Pretra "PyTorch SDPA requirements in Transformers are not met. Please install torch>=2.1.1." ) - # VLMs have to follow the sdpa attr of its sub-configs - if hasattr(config, "text_config") and hasattr(config, "vision_config"): - if ( - config.text_config._attn_implementation == "sdpa" - or config.vision_config._attn_implementation == "sdpa" - ): - config._attn_implementation = "sdpa" + # VLM/Encoder-Decoder etc. have to follow the sdpa attr of its sub-configs + sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} + if sub_configs: + sub_config_attentions = {sub_config._attn_implementation for key, sub_config in sub_configs.items()} + config._attn_implementation = "sdpa" if "sdpa" in sub_config_attentions else config._attn_implementation if not is_torch_sdpa_available() or not cls._supports_sdpa: return config diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py index 979bd69de9be..ced1a48a45a4 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py @@ -190,10 +190,12 @@ def __init__( super().__init__(config) if encoder is None: - encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation) + encoder = AutoModel.from_config(config.encoder, attn_implementation=config.encoder._attn_implementation) if decoder is None: - decoder = AutoModelForCausalLM.from_config(config.decoder, attn_implementation=config._attn_implementation) + decoder = AutoModelForCausalLM.from_config( + config.decoder, attn_implementation=config.decoder._attn_implementation + ) self.encoder = encoder self.decoder = decoder diff --git a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py index e5bc88d5bfb2..c354e6bf0f55 100644 --- a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py +++ b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py @@ -20,6 +20,7 @@ from datasets import load_dataset from huggingface_hub import hf_hub_download from packaging import version +from parameterized import parameterized from transformers import DonutProcessor, NougatProcessor, TrOCRProcessor from transformers.testing_utils import ( @@ -27,17 +28,26 @@ require_nltk, require_sentencepiece, require_torch, + require_torch_sdpa, require_vision, slow, to_2tuple, torch_device, ) -from transformers.utils import cached_property, is_torch_available, is_vision_available +from transformers.utils import ( + cached_property, + is_torch_available, + is_torch_bf16_available_on_device, + is_torch_fp16_available_on_device, + is_vision_available, +) from ...test_modeling_common import floats_tensor, ids_tensor, random_attention_mask from ..bart.test_modeling_bart import BartModelTester from ..bert.test_modeling_bert import BertModelTester from ..deit.test_modeling_deit import DeiTModelTester +from ..donut.test_modeling_donut_swin import DonutSwinModelTester +from ..gpt2.test_modeling_gpt2 import GPT2ModelTester from ..layoutlmv3.test_modeling_layoutlmv3 import LayoutLMv3ModelTester from ..swin.test_modeling_swin import SwinModelTester from ..trocr.test_modeling_trocr import TrOCRStandaloneDecoderModelTester @@ -53,6 +63,8 @@ BartForCausalLM, BertLMHeadModel, DeiTModel, + DonutSwinModel, + GPT2LMHeadModel, LayoutLMv3Model, SwinModel, TrOCRForCausalLM, @@ -72,6 +84,9 @@ @require_torch class EncoderDecoderMixin: + has_attentions: bool = False + supports_sdpa: bool = False + def get_encoder_decoder_model(self, config, decoder_config): pass @@ -374,6 +389,90 @@ def test_real_model_save_load_from_pretrained(self): max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @require_torch_sdpa + @slow + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self.supports_sdpa: + self.skipTest("SDPA is not supported") + + if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): + self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") + + if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): + self.skipTest( + f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" + ) + + # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. + if torch_dtype == "float16": + torch_dtype = torch.float16 + elif torch_dtype == "bfloat16": + torch_dtype = torch.bfloat16 + elif torch_dtype == "float32": + torch_dtype = torch.float32 + + inputs_dict = self.prepare_config_and_inputs() + encoder_config, decoder_config = inputs_dict["config"], inputs_dict["decoder_config"] + config = VisionEncoderDecoderConfig.from_encoder_decoder_configs( + encoder_config=encoder_config, decoder_config=decoder_config + ) + model = VisionEncoderDecoderModel(config=config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = VisionEncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = model_sdpa.eval().to(torch_device) + + # see https://github.com/huggingface/transformers/pull/32238 + # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not + # In this case we get SDPA by default if it `_supports_Sdpa` else fallback to "eager" + encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + general_attn = "sdpa" # we didn't skip test, so this model supports sdpa for sure + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa.config.encoder._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + + # Also test that nothing break if we request SDPA explicitly + # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible + # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA + # Checking error is out-of-scope of this test + model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa") + model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) + + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) + + model_eager = VisionEncoderDecoderModel.from_pretrained( + tmpdirname, + torch_dtype=torch_dtype, + attn_implementation="eager", + ) + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config.encoder._attn_implementation == "eager") + self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + @require_torch class DeiT2RobertaModelTest(EncoderDecoderMixin, unittest.TestCase): @@ -497,6 +596,9 @@ def prepare_config_and_inputs(self): @require_torch class ViT2BertModelTest(EncoderDecoderMixin, unittest.TestCase): + has_attentions = True + supports_sdpa = True # one submodel support SDPA + def get_pretrained_model_and_inputs(self): model = VisionEncoderDecoderModel.from_encoder_decoder_pretrained( "hf-internal-testing/tiny-random-vit", "hf-internal-testing/tiny-bert" @@ -649,6 +751,9 @@ def test_real_model_save_load_from_pretrained(self): @require_torch class ViT2TrOCR(EncoderDecoderMixin, unittest.TestCase): + has_attentions = True + supports_sdpa = True # one submodel support SDPA + def get_encoder_decoder_model(self, config, decoder_config): encoder_model = ViTModel(config).eval() decoder_model = TrOCRForCausalLM(decoder_config).eval() @@ -803,6 +908,241 @@ def check_encoder_decoder_model_generate(self, config, decoder_config, pixel_val def test_real_model_save_load_from_pretrained(self): pass +@require_torch +class VIT2GPT2Test(EncoderDecoderMixin, unittest.TestCase): + has_attentions = True + supports_sdpa = True # both submodels support SDPA + + def get_encoder_decoder_model(self, config, decoder_config): + encoder_model = ViTModel(config).eval() + decoder_model = GPT2LMHeadModel(decoder_config).eval() + return encoder_model, decoder_model + + def prepare_config_and_inputs(self): + model_tester_encoder = ViTModelTester(self, batch_size=13) + model_tester_decoder = GPT2ModelTester(self, batch_size=13, hidden_size=32, max_position_embeddings=512) + encoder_config_and_inputs = model_tester_encoder.prepare_config_and_inputs() + decoder_config_and_inputs = model_tester_decoder.prepare_config_and_inputs() + config, pixel_values, labels = encoder_config_and_inputs + ( + decoder_config, + decoder_input_ids, + decoder_attention_mask, + decoder_head_mask, + decoder_token_type_ids, + mc_token_ids, + sequence_labels, + token_labels, + choice_labels, + ) = decoder_config_and_inputs + + # make sure that cross attention layers are added + decoder_config.add_cross_attention = True + # disable cache for now + decoder_config.use_cache = False + return { + "config": config, + "pixel_values": pixel_values, + "decoder_config": decoder_config, + "decoder_input_ids": decoder_input_ids, + "decoder_attention_mask": decoder_attention_mask, + "decoder_head_mask": decoder_head_mask, + "labels": decoder_input_ids, + } + + def check_encoder_decoder_model_output_attentions( + self, + config, + decoder_config, + decoder_input_ids, + decoder_attention_mask, + pixel_values, + labels=None, + **kwargs, + ): + # make the decoder inputs a different shape from the encoder inputs to harden the test + decoder_input_ids = decoder_input_ids[:, :-1] + decoder_attention_mask = decoder_attention_mask[:, :-1] + encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config) + enc_dec_model = VisionEncoderDecoderModel(encoder=encoder_model, decoder=decoder_model) + enc_dec_model.to(torch_device) + outputs_encoder_decoder = enc_dec_model( + pixel_values=pixel_values, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + output_attentions=True, + **kwargs, + ) + + encoder_attentions = outputs_encoder_decoder["encoder_attentions"] + self.assertEqual(len(encoder_attentions), config.num_hidden_layers) + + seq_len = (encoder_model.config.image_size // encoder_model.config.patch_size) ** 2 + 1 + + decoder_attentions = outputs_encoder_decoder["decoder_attentions"] + num_decoder_layers = ( + decoder_config.num_decoder_layers + if hasattr(decoder_config, "num_decoder_layers") + else decoder_config.num_hidden_layers + ) + self.assertEqual(len(decoder_attentions), num_decoder_layers) + + self.assertEqual( + decoder_attentions[0].shape[-3:], + (decoder_config.num_attention_heads, decoder_input_ids.shape[-1], decoder_input_ids.shape[-1]), + ) + + cross_attentions = outputs_encoder_decoder["cross_attentions"] + self.assertEqual(len(cross_attentions), num_decoder_layers) + + cross_attention_input_seq_len = decoder_input_ids.shape[-1] + self.assertEqual( + cross_attentions[0].shape[-3:], + (decoder_config.num_attention_heads, cross_attention_input_seq_len, seq_len), # 4 6 16 + ) + + def check_encoder_decoder_model_generate(self, config, decoder_config, pixel_values=None, **kwargs): + encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config) + enc_dec_model = VisionEncoderDecoderModel(encoder=encoder_model, decoder=decoder_model) + + # Generate until max length + if hasattr(enc_dec_model.config, "eos_token_id"): + enc_dec_model.config.eos_token_id = None + if hasattr(enc_dec_model.config, "decoder") and hasattr(enc_dec_model.config.decoder, "eos_token_id"): + enc_dec_model.config.decoder.eos_token_id = None + if hasattr(enc_dec_model.generation_config, "eos_token_id"): + enc_dec_model.generation_config.eos_token_id = None + enc_dec_model.to(torch_device) + + generated_output = enc_dec_model.generate( + pixel_values=pixel_values, + decoder_start_token_id=enc_dec_model.config.decoder.bos_token_id, + **kwargs, + ) + self.assertEqual(generated_output.shape, (pixel_values.shape[0],) + (decoder_config.max_length,)) + + @unittest.skip(reason="VIT2GPT2 also has an integration test for testinf save-load") + def test_real_model_save_load_from_pretrained(self): + pass + + +@require_torch +class Donut2GPT2Test(EncoderDecoderMixin, unittest.TestCase): + has_attentions = True + supports_sdpa = True # one submodel (GPT2) support SDPA + + def get_encoder_decoder_model(self, config, decoder_config): + encoder_model = DonutSwinModel(config).eval() + decoder_model = GPT2LMHeadModel(decoder_config).eval() + return encoder_model, decoder_model + + def prepare_config_and_inputs(self): + model_tester_encoder = DonutSwinModelTester(self, batch_size=13) + model_tester_decoder = GPT2ModelTester(self, batch_size=13, hidden_size=32, max_position_embeddings=512) + encoder_config_and_inputs = model_tester_encoder.prepare_config_and_inputs() + decoder_config_and_inputs = model_tester_decoder.prepare_config_and_inputs() + config, pixel_values, labels = encoder_config_and_inputs + ( + decoder_config, + decoder_input_ids, + decoder_attention_mask, + decoder_head_mask, + decoder_token_type_ids, + mc_token_ids, + sequence_labels, + token_labels, + choice_labels, + ) = decoder_config_and_inputs + + # make sure that cross attention layers are added + decoder_config.add_cross_attention = True + # disable cache for now + decoder_config.use_cache = False + return { + "config": config, + "pixel_values": pixel_values, + "decoder_config": decoder_config, + "decoder_input_ids": decoder_input_ids, + "decoder_attention_mask": decoder_attention_mask, + "decoder_head_mask": decoder_head_mask, + "labels": decoder_input_ids, + } + + def check_encoder_decoder_model_output_attentions( + self, + config, + decoder_config, + decoder_input_ids, + decoder_attention_mask, + pixel_values, + labels=None, + **kwargs, + ): + # make the decoder inputs a different shape from the encoder inputs to harden the test + decoder_input_ids = decoder_input_ids[:, :-1] + decoder_attention_mask = decoder_attention_mask[:, :-1] + encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config) + enc_dec_model = VisionEncoderDecoderModel(encoder=encoder_model, decoder=decoder_model) + enc_dec_model.to(torch_device) + outputs_encoder_decoder = enc_dec_model( + pixel_values=pixel_values, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + output_attentions=True, + **kwargs, + ) + + encoder_attentions = outputs_encoder_decoder["encoder_attentions"] + self.assertEqual(len(encoder_attentions), config.num_hidden_layers) + + seq_len = encoder_model.config.image_size // encoder_model.config.patch_size + + decoder_attentions = outputs_encoder_decoder["decoder_attentions"] + num_decoder_layers = ( + decoder_config.num_decoder_layers + if hasattr(decoder_config, "num_decoder_layers") + else decoder_config.num_hidden_layers + ) + self.assertEqual(len(decoder_attentions), num_decoder_layers) + + self.assertEqual( + decoder_attentions[0].shape[-3:], + (decoder_config.num_attention_heads, decoder_input_ids.shape[-1], decoder_input_ids.shape[-1]), + ) + + cross_attentions = outputs_encoder_decoder["cross_attentions"] + self.assertEqual(len(cross_attentions), num_decoder_layers) + + cross_attention_input_seq_len = decoder_input_ids.shape[-1] + self.assertEqual( + cross_attentions[0].shape[-3:], + (decoder_config.num_attention_heads, cross_attention_input_seq_len, seq_len), # 4 6 16 + ) + + def check_encoder_decoder_model_generate(self, config, decoder_config, pixel_values=None, **kwargs): + encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config) + enc_dec_model = VisionEncoderDecoderModel(encoder=encoder_model, decoder=decoder_model) + + # Generate until max length + if hasattr(enc_dec_model.config, "eos_token_id"): + enc_dec_model.config.eos_token_id = None + if hasattr(enc_dec_model.config, "decoder") and hasattr(enc_dec_model.config.decoder, "eos_token_id"): + enc_dec_model.config.decoder.eos_token_id = None + if hasattr(enc_dec_model.generation_config, "eos_token_id"): + enc_dec_model.generation_config.eos_token_id = None + enc_dec_model.to(torch_device) + + generated_output = enc_dec_model.generate( + pixel_values=pixel_values, + decoder_start_token_id=enc_dec_model.config.decoder.bos_token_id, + **kwargs, + ) + self.assertEqual(generated_output.shape, (pixel_values.shape[0],) + (decoder_config.max_length,)) + + @unittest.skip(reason="Donut has an Integration test for that") + def test_real_model_save_load_from_pretrained(self): + pass + @require_vision @require_torch From 52e77b34c4ddb1b5b5863e761c833327820e0b10 Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 11:51:17 +0200 Subject: [PATCH 14/68] codestyle --- .../test_modeling_vision_encoder_decoder.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py index c354e6bf0f55..5a3ae65a822d 100644 --- a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py +++ b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py @@ -432,7 +432,8 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): # In this case we get SDPA by default if it `_supports_Sdpa` else fallback to "eager" encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - general_attn = "sdpa" # we didn't skip test, so this model supports sdpa for sure + + # We didn't skip test, so this model supports sdpa for sure in general config self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.config.encoder._attn_implementation == encoder_attn) self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) @@ -441,9 +442,11 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA # Checking error is out-of-scope of this test - model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa") + model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) @@ -597,7 +600,7 @@ def prepare_config_and_inputs(self): @require_torch class ViT2BertModelTest(EncoderDecoderMixin, unittest.TestCase): has_attentions = True - supports_sdpa = True # one submodel support SDPA + supports_sdpa = True # one submodel support SDPA def get_pretrained_model_and_inputs(self): model = VisionEncoderDecoderModel.from_encoder_decoder_pretrained( @@ -752,7 +755,7 @@ def test_real_model_save_load_from_pretrained(self): @require_torch class ViT2TrOCR(EncoderDecoderMixin, unittest.TestCase): has_attentions = True - supports_sdpa = True # one submodel support SDPA + supports_sdpa = True # one submodel support SDPA def get_encoder_decoder_model(self, config, decoder_config): encoder_model = ViTModel(config).eval() @@ -908,10 +911,11 @@ def check_encoder_decoder_model_generate(self, config, decoder_config, pixel_val def test_real_model_save_load_from_pretrained(self): pass + @require_torch class VIT2GPT2Test(EncoderDecoderMixin, unittest.TestCase): has_attentions = True - supports_sdpa = True # both submodels support SDPA + supports_sdpa = True # both submodels support SDPA def get_encoder_decoder_model(self, config, decoder_config): encoder_model = ViTModel(config).eval() @@ -1020,7 +1024,7 @@ def check_encoder_decoder_model_generate(self, config, decoder_config, pixel_val **kwargs, ) self.assertEqual(generated_output.shape, (pixel_values.shape[0],) + (decoder_config.max_length,)) - + @unittest.skip(reason="VIT2GPT2 also has an integration test for testinf save-load") def test_real_model_save_load_from_pretrained(self): pass @@ -1029,7 +1033,7 @@ def test_real_model_save_load_from_pretrained(self): @require_torch class Donut2GPT2Test(EncoderDecoderMixin, unittest.TestCase): has_attentions = True - supports_sdpa = True # one submodel (GPT2) support SDPA + supports_sdpa = True # one submodel (GPT2) support SDPA def get_encoder_decoder_model(self, config, decoder_config): encoder_model = DonutSwinModel(config).eval() From 0feec1e406a2fa874aaf3113668d647e3fcc6a0c Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 12:13:05 +0200 Subject: [PATCH 15/68] add tests for idefics --- .../models/idefics2/modeling_idefics2.py | 6 +- tests/models/idefics/test_modeling_idefics.py | 76 ++++++++++++++-- .../models/idefics2/test_modeling_idefics2.py | 89 +++++++++++++++++-- 3 files changed, 152 insertions(+), 19 deletions(-) diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index e57be17f5096..7e88687c0e7b 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -1135,9 +1135,9 @@ def _autoset_attn_implementation( ) # autoset-attn calls recursively all sub-configs (text-config, vision-config) # and sets attn implementation if the config can be mapped bu auto-model - # Idefics2 vision config can't be mapped automcatically so we set it manually here - # We cant set vision attn same as general attn, because the general one can be sdpa if at - # least one sub-module (in this case LLM) supports sdpa + # Idefics2 vision config can't be mapped automatically so we set it manually here + # We can't set vision attn same as general attn, because the general attr will be sdpa if at + # least one sub-module (in this case LLM) supports sdpa, and we know vision/perceiver doesn't support yet if hasattr(config, "vision_config"): config.vision_config._attn_implementation = ( config._attn_implementation if config._attn_implementation != "sdpa" else "eager" diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index 0197ebcaff53..1cf085b6ee66 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -14,6 +14,7 @@ # limitations under the License. """Testing suite for the PyTorch Idefics model.""" +import tempfile import unittest from parameterized import parameterized @@ -29,7 +30,7 @@ slow, torch_device, ) -from transformers.utils import cached_property +from transformers.utils import cached_property, is_torch_bf16_available_on_device, is_torch_fp16_available_on_device from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask @@ -312,12 +313,6 @@ def prepare_config_and_inputs_for_common(self): def prepare_pixel_values(self): return floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) - @require_torch_sdpa - @slow - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - self.skipTest(reason="Idefics has a hard requirement on SDPA, skipping this test") - @unittest.skipIf(not is_torch_greater_or_equal_than_2_0, reason="pytorch 2.0 or higher is required") @require_torch @@ -571,11 +566,74 @@ def test_model_from_pretrained(self): model = IdeficsModel.from_pretrained(model_name) self.assertIsNotNone(model) + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) def test_eager_matches_sdpa_inference(self, torch_dtype: str): - self.skipTest(reason="Idefics has a hard requirement on SDPA, skipping this test") + if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): + self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") + + if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): + self.skipTest( + f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" + ) + + # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. + if torch_dtype == "float16": + torch_dtype = torch.float16 + elif torch_dtype == "bfloat16": + torch_dtype = torch.bfloat16 + elif torch_dtype == "float32": + torch_dtype = torch.float32 + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = model_sdpa.eval().to(torch_device) + + # see https://github.com/huggingface/transformers/pull/32238 + # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not + # In this case we get SDPA by default if it `_supports_sdpa` else fallback to "eager" + # and we know that Idefics's perceiver and vision models are simple nn.Modules wo SDPA + perceiver_attn = "eager" + vision_attn = "eager" + + # We also know this model supports sdpa for sure in general config + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) + self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + + # Also test that nothing break if we request SDPA explicitly + # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible + # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA + # Checking error is out-of-scope of this test + model_sdpa_explicit = model_class.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) + model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) + + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa_explicit.config.perceiver_config._attn_implementation == perceiver_attn) + self.assertTrue(model_sdpa_explicit.config.vision_config._attn_implementation == vision_attn) + + model_eager = model_class.from_pretrained( + tmpdirname, + torch_dtype=torch_dtype, + attn_implementation="eager", + ) + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config.perceiver_config._attn_implementation == "eager") + self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") @unittest.skipIf(not is_torch_greater_or_equal_than_2_0, reason="pytorch 2.0 or higher is required") diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 8386021a7ebb..9b7e4b50cedb 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -16,10 +16,12 @@ import copy import gc +import tempfile import unittest from io import BytesIO import requests +from parameterized import parameterized from transformers import ( AutoProcessor, @@ -29,7 +31,8 @@ is_torch_available, is_vision_available, ) -from transformers.testing_utils import require_bitsandbytes, require_torch, slow, torch_device +from transformers.testing_utils import require_bitsandbytes, require_torch, require_torch_sdpa, slow, torch_device +from transformers.utils import is_torch_bf16_available_on_device, is_torch_fp16_available_on_device from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester @@ -172,12 +175,6 @@ class Idefics2ModelTest(ModelTesterMixin, unittest.TestCase): test_resize_embeddings = True test_head_masking = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because LM/vision models used in tests dont support SDPA - supports_sdpa = False - def setUp(self): self.model_tester = Idefics2VisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=Idefics2Config, has_text_modality=False) @@ -324,6 +321,84 @@ def test_resize_embeddings_untied(self): # Check that the model can still do a forward pass successfully (every parameter should be resized) model(**self._prepare_for_class(inputs_dict, model_class)) + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @require_torch_sdpa + @slow + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): + self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") + + if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): + self.skipTest( + f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" + ) + + # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. + if torch_dtype == "float16": + torch_dtype = torch.float16 + elif torch_dtype == "bfloat16": + torch_dtype = torch.bfloat16 + elif torch_dtype == "float32": + torch_dtype = torch.float32 + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = model_sdpa.eval().to(torch_device) + + # see https://github.com/huggingface/transformers/pull/32238 + # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not + # In this case we know that Idefics2 has 'eager' for all perceiver/vision models and that + # they are not PretrainedModel so we cannot check `_supports_sdpa` cls attr but only set it here + perceiver_attn = "eager" + vision_attn = "eager" + + # We didn't skip test, so this model supports sdpa for sure in general config + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) + self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + + # Also test that nothing break if we request SDPA explicitly + # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible + # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA + # Checking error is out-of-scope of this test + model_sdpa_explicit = model_class.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) + model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) + + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) + self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + + model_eager = model_class.from_pretrained( + tmpdirname, + torch_dtype=torch_dtype, + attn_implementation="eager", + ) + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == "eager") + self.assertTrue(model_sdpa.config.vision_config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + @require_torch class Idefics2ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTesterMixin, unittest.TestCase): From 7b610963196ba8c76fb0a01d3dd17c4518bc39da Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 12:28:57 +0200 Subject: [PATCH 16/68] modify general test for VLMs --- tests/models/llava/test_modeling_llava.py | 1 + tests/models/llava_next/test_modeling_llava_next.py | 1 + .../llava_next_video/test_modeling_llava_next_video.py | 1 + tests/models/paligemma/test_modeling_paligemma.py | 1 + tests/models/video_llava/test_modeling_video_llava.py | 1 + tests/models/vipllava/test_modeling_vipllava.py | 1 + tests/test_modeling_common.py | 10 ++++++++++ 7 files changed, 16 insertions(+) diff --git a/tests/models/llava/test_modeling_llava.py b/tests/models/llava/test_modeling_llava.py index 5e9e336f554e..07ebcb81c961 100644 --- a/tests/models/llava/test_modeling_llava.py +++ b/tests/models/llava/test_modeling_llava.py @@ -178,6 +178,7 @@ class LlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase """ all_model_classes = (LlavaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (LlavaForConditionalGeneration,) if is_torch_available() else () pipeline_model_mapping = {"image-to-text": LlavaForConditionalGeneration} if is_torch_available() else {} test_pruning = False test_head_masking = False diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index 3234f001f951..45a3607c012c 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -214,6 +214,7 @@ class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, GenerationTes """ all_model_classes = (LlavaNextForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (LlavaNextForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False is_multimodal = True diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index d22e8a1dc824..42184fdc7d20 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -229,6 +229,7 @@ class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, Generati """ all_model_classes = (LlavaNextVideoForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (LlavaNextVideoForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False is_multimodal = True diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index ce1d5eb5686a..beb276402685 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -176,6 +176,7 @@ class PaliGemmaForConditionalGenerationModelTest(ModelTesterMixin, unittest.Test """ all_model_classes = (PaliGemmaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (PaliGemmaForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_pruning = False test_torchscript = False diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index 10aeac71256e..c3e544d75a9c 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -196,6 +196,7 @@ class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTe """ all_model_classes = (VideoLlavaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (VideoLlavaForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_pruning = False test_resize_embeddings = True diff --git a/tests/models/vipllava/test_modeling_vipllava.py b/tests/models/vipllava/test_modeling_vipllava.py index 51cf66d856f7..63e9a2337bd1 100644 --- a/tests/models/vipllava/test_modeling_vipllava.py +++ b/tests/models/vipllava/test_modeling_vipllava.py @@ -158,6 +158,7 @@ class VipLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestC """ all_model_classes = (VipLlavaForConditionalGeneration,) if is_torch_available() else () + all_generative_model_classes = (VipLlavaForConditionalGeneration,) if is_torch_available() else () fx_compatible = False test_pruning = False test_resize_embeddings = True diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index cc4abaf88e57..0e863f2d1ec3 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3778,6 +3778,14 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_sdpa.eval().to(torch_device) self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + if self.is_multimodal: + vision_supports_sdpa = ( + model.image_tower._supports_sdpa + if hasattr(model_sdpa, "image_tower") + else model.vision_tower._supports_sdpa + ) + vision_attn = "sdpa" if vision_supports_sdpa else "eager" + self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) model_eager = model_class.from_pretrained( tmpdirname, @@ -3787,6 +3795,8 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_eager = model_eager.eval().to(torch_device) self.assertTrue(model_eager.config._attn_implementation == "eager") + if self.is_multimodal: + self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ From 9d15024f316d1e499e7fd18c2d2e20dd33b39c2b Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 12:51:22 +0200 Subject: [PATCH 17/68] no generation test for vlm yet! --- .../models/llava_next_video/test_modeling_llava_next_video.py | 3 +-- tests/models/video_llava/test_modeling_video_llava.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 42184fdc7d20..887124ee3c2f 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -34,7 +34,6 @@ torch_device, ) -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -223,7 +222,7 @@ def create_and_check_llava_next_video_model_fp16_autocast_forward( @require_torch -class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): """ Model tester for `LlavaNextVideoForConditionalGeneration`. """ diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index c3e544d75a9c..d27c4cfe39cf 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -30,7 +30,6 @@ ) from transformers.testing_utils import require_bitsandbytes, require_torch, require_torch_gpu, slow, torch_device -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor @@ -190,7 +189,7 @@ def prepare_config_and_inputs_for_batched_test(self): @require_torch -class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): """ Model tester for `VideoLlavaForConditionalGeneration`. """ From 4e20af650795cc17540e38d21712f5b175b53bed Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 13:08:17 +0200 Subject: [PATCH 18/68] no generation test here also --- tests/models/llava_next/test_modeling_llava_next.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index 45a3607c012c..d57e2a88ae84 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -208,7 +208,7 @@ def create_and_check_llava_next_model_fp16_autocast_forward( @require_torch -class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): +class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): """ Model tester for `LlavaNextForConditionalGeneration`. """ From 1c6435d610811b9d345f53f662213fa5e5e8e229 Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 13:26:20 +0200 Subject: [PATCH 19/68] wanr in VIT-SDPA if output attn --- src/transformers/models/vit/modeling_vit.py | 18 +++++++++++++++++- .../llava_next/test_modeling_llava_next.py | 1 - 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index 7897555a6baf..7cd070df266e 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -241,8 +241,24 @@ def __init__(self, config: ViTConfig) -> None: self.attention_probs_dropout_prob = config.attention_probs_dropout_prob def forward( - self, hidden_states, head_mask: Optional[torch.Tensor] = None, output_attentions: bool = False + self, + hidden_states: torch.FloatTensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: + if output_attentions or head_mask is not None: + logger.warning_once( + "`VITdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " + "`output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but " + "specifying the manual implementation will be required from Transformers version v5.0.0 onwards. " + 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + head_mask=head_mask, + output_attentions=output_attentions, + ) + mixed_query_layer = self.query(hidden_states) key_layer = self.transpose_for_scores(self.key(hidden_states)) diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index d57e2a88ae84..e1d71f2704dc 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -34,7 +34,6 @@ torch_device, ) -from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, From ea71d89f11ee7ef00cc9962f9b88bd3bfe9e59ba Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 13:33:10 +0200 Subject: [PATCH 20/68] add more tests --- .../test_modeling_encoder_decoder.py | 100 +++++++++++++++++- .../test_modeling_vision_encoder_decoder.py | 2 +- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py index 5e5263b6afb9..9bb9982572c6 100644 --- a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py +++ b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py @@ -17,8 +17,18 @@ import tempfile import unittest +from parameterized import parameterized + from transformers import is_torch_available, logging -from transformers.testing_utils import CaptureLogger, require_deterministic_for_xpu, require_torch, slow, torch_device +from transformers.testing_utils import ( + CaptureLogger, + require_deterministic_for_xpu, + require_torch, + require_torch_sdpa, + slow, + torch_device, +) +from transformers.utils import is_torch_bf16_available_on_device, is_torch_fp16_available_on_device from ...test_modeling_common import ids_tensor from ..bart.test_modeling_bart import BartStandaloneDecoderModelTester @@ -54,6 +64,8 @@ @require_torch class EncoderDecoderMixin: + supports_sdpa = False + def get_encoder_decoder_model(self, config, decoder_config): raise NotImplementedError @@ -670,6 +682,90 @@ def test_real_model_save_load_from_pretrained(self): max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @require_torch_sdpa + @slow + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + if not self.supports_sdpa: + self.skipTest("SDPA is not supported") + + if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): + self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") + + if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): + self.skipTest( + f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" + ) + + # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. + if torch_dtype == "float16": + torch_dtype = torch.float16 + elif torch_dtype == "bfloat16": + torch_dtype = torch.bfloat16 + elif torch_dtype == "float32": + torch_dtype = torch.float32 + + inputs_dict = self.prepare_config_and_inputs() + encoder_config, decoder_config = inputs_dict["config"], inputs_dict["decoder_config"] + config = EncoderDecoderConfig.from_encoder_decoder_configs( + encoder_config=encoder_config, decoder_config=decoder_config + ) + model = EncoderDecoderModel(config=config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = EncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = model_sdpa.eval().to(torch_device) + + # see https://github.com/huggingface/transformers/pull/32238 + # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not + # In this case we get SDPA by default if it `_supports_Sdpa` else fallback to "eager" + encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + + # We didn't skip test, so this model supports sdpa for sure in general config + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa.config.encoder._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + + # Also test that nothing break if we request SDPA explicitly + # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible + # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA + # Checking error is out-of-scope of this test + model_sdpa_explicit = EncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) + model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) + + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) + + model_eager = EncoderDecoderModel.from_pretrained( + tmpdirname, + torch_dtype=torch_dtype, + attn_implementation="eager", + ) + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config.encoder._attn_implementation == "eager") + self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa: + raise ValueError("The SDPA model should have SDPA attention layers") + @require_torch class BertEncoderDecoderModelTest(EncoderDecoderMixin, unittest.TestCase): @@ -949,6 +1045,8 @@ def get_pretrained_model(self): @require_torch class GPT2EncoderDecoderModelTest(EncoderDecoderMixin, unittest.TestCase): + supports_sdpa = True + def get_encoder_decoder_model(self, config, decoder_config): encoder_model = BertModel(config) decoder_model = GPT2LMHeadModel(decoder_config) diff --git a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py index 5a3ae65a822d..2f341f3df3ca 100644 --- a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py +++ b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py @@ -473,7 +473,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: has_sdpa = True break - if not has_sdpa and model_sdpa.config.model_type != "falcon": + if not has_sdpa: raise ValueError("The SDPA model should have SDPA attention layers") From 17f9e69f7db6a0a56799a89816aebde153812087 Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 15:15:56 +0200 Subject: [PATCH 21/68] user can pass dict as attn impl --- src/transformers/modeling_utils.py | 32 +++++++++++++------ .../models/llava/modeling_llava.py | 1 + 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index f3adfa16c10a..a3ab77d9aeda 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1502,7 +1502,11 @@ def _autoset_attn_implementation( ' We recommend to just use `attn_implementation="flash_attention_2"` when loading the model.' ) - if config._attn_implementation not in ["eager", "sdpa", "flash_attention_2"]: + if not isinstance(config._attn_implementation, dict) and config._attn_implementation not in [ + "eager", + "sdpa", + "flash_attention_2", + ]: message = f'Specified `attn_implementation="{config._attn_implementation}"` is not supported. The only possible arguments are `attn_implementation="eager"` (manual attention implementation)' if cls._supports_flash_attn_2: message += ', `"attn_implementation=flash_attention_2"` (implementation using flash attention 2)' @@ -1513,21 +1517,22 @@ def _autoset_attn_implementation( # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available. requested_attn_implementation = config._attn_implementation_internal - # MultiModal-LLM/Encoder-Decoder related hack: since they consist of two or might be more sub-configs + # MultiModal-LLM/Encoder-Decoder related block: since they consist of two or might be more sub-configs # we have to check and dispatch SDPA to each sub-config, in case any of them support it. # If one sub-model supports SDPA while other doesn't, an error will be raised following the - # typical SDPA-dispatch path (i.e. if hard_check) + # typical SDPA-dispatch path (i.e. if hard_check). Same goes for FA2. sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} if sub_configs: - # Set to None to avoid hard_check errors, because generalist model's `_support_sdpa` attr is usually `False` by default - # while sub-configs can still be set to `True` - requested_attn_implementation = ( - None if requested_attn_implementation == "sdpa" else requested_attn_implementation - ) for key, sub_config in sub_configs.items(): sub_model_cls = MODEL_MAPPING.get(type(sub_config), None) if sub_model_cls is not None: - sub_config._attn_implementation = requested_attn_implementation + # User can pass attn_implementation={"vision_config": "sdpa", "text_config": "eager"} + # as well as attn_implementation="sdpa", which means sdpa in all sub-configs + if isinstance(requested_attn_implementation, dict) and key in requested_attn_implementation: + sub_config._attn_implementation = requested_attn_implementation[key] + else: + sub_config._attn_implementation = requested_attn_implementation + sub_model_cls._autoset_attn_implementation( sub_config, use_flash_attention_2=use_flash_attention_2, @@ -1537,6 +1542,13 @@ def _autoset_attn_implementation( ) setattr(config, key, sub_config) + # Set to eager to avoid hard_check errors that cls does not support sdpa/FA2, because generalist model's + # cls attr should be set to False by default. Sub-configs cls attr can still be set to True + # If any sub-config doesnt support requested attn, we'll catch that when dispatching the sub-config above + requested_attn_implementation = ( + None if requested_attn_implementation == "sdpa" else requested_attn_implementation + ) + if use_flash_attention_2: logger.warning_once( 'The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation="flash_attention_2"` instead.' @@ -1567,6 +1579,8 @@ def _autoset_attn_implementation( "Using the `SDPA` attention implementation on multi-gpu setup with ROCM may lead to performance issues due to the FA backend. Disabling it to use alternative backends." ) torch.backends.cuda.enable_flash_sdp(False) + elif isinstance(requested_attn_implementation, dict): + config._attn_implementation = requested_attn_implementation else: config._attn_implementation = "eager" diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 9c9bcf95dcb2..8022d23e958f 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -238,6 +238,7 @@ def __init__(self, config: LlavaConfig): config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 + print(config.vision_config._attn_implementation, config.text_config._attn_implementation) self.post_init() def get_input_embeddings(self): From 140f2228e49f1b3baee6a23fd5452cbebefd8f5f Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 31 Jul 2024 15:34:55 +0200 Subject: [PATCH 22/68] repo consistency --- src/transformers/modeling_utils.py | 1 + .../modeling_audio_spectrogram_transformer.py | 18 +++++++++++++++++- src/transformers/models/deit/modeling_deit.py | 18 +++++++++++++++++- .../models/vipllava/modeling_vipllava.py | 1 + src/transformers/models/vit/modeling_vit.py | 2 +- .../models/vit_mae/modeling_vit_mae.py | 18 +++++++++++++++++- .../models/vit_msn/modeling_vit_msn.py | 18 +++++++++++++++++- .../models/yolos/modeling_yolos.py | 18 +++++++++++++++++- 8 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index a3ab77d9aeda..b13c86605b92 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1533,6 +1533,7 @@ def _autoset_attn_implementation( else: sub_config._attn_implementation = requested_attn_implementation + print(type(sub_config), requested_attn_implementation) sub_model_cls._autoset_attn_implementation( sub_config, use_flash_attention_2=use_flash_attention_2, diff --git a/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py b/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py index beb249202b96..491c6ce16461 100644 --- a/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py +++ b/src/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py @@ -176,8 +176,24 @@ def __init__(self, config: ASTConfig) -> None: self.attention_probs_dropout_prob = config.attention_probs_dropout_prob def forward( - self, hidden_states, head_mask: Optional[torch.Tensor] = None, output_attentions: bool = False + self, + hidden_states: torch.FloatTensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: + if output_attentions or head_mask is not None: + logger.warning_once( + "`ASTSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " + "`output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but " + "specifying the manual implementation will be required from Transformers version v5.0.0 onwards. " + 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + head_mask=head_mask, + output_attentions=output_attentions, + ) + mixed_query_layer = self.query(hidden_states) key_layer = self.transpose_for_scores(self.key(hidden_states)) diff --git a/src/transformers/models/deit/modeling_deit.py b/src/transformers/models/deit/modeling_deit.py index 0f5bef5710a8..fe033952497b 100644 --- a/src/transformers/models/deit/modeling_deit.py +++ b/src/transformers/models/deit/modeling_deit.py @@ -243,8 +243,24 @@ def __init__(self, config: DeiTConfig) -> None: self.attention_probs_dropout_prob = config.attention_probs_dropout_prob def forward( - self, hidden_states, head_mask: Optional[torch.Tensor] = None, output_attentions: bool = False + self, + hidden_states: torch.FloatTensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: + if output_attentions or head_mask is not None: + logger.warning_once( + "`DeiTSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " + "`output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but " + "specifying the manual implementation will be required from Transformers version v5.0.0 onwards. " + 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + head_mask=head_mask, + output_attentions=output_attentions, + ) + mixed_query_layer = self.query(hidden_states) key_layer = self.transpose_for_scores(self.key(hidden_states)) diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 58ebf2db2dea..2108b1a73ba5 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -243,6 +243,7 @@ def __init__(self, config: VipLlavaConfig): config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 + print(config.vision_config._attn_implementation, config.text_config._attn_implementation) self.post_init() def get_input_embeddings(self): diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index 7cd070df266e..9fbf121de6ca 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -248,7 +248,7 @@ def forward( ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: if output_attentions or head_mask is not None: logger.warning_once( - "`VITdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " + "`ViTSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " "`output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but " "specifying the manual implementation will be required from Transformers version v5.0.0 onwards. " 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' diff --git a/src/transformers/models/vit_mae/modeling_vit_mae.py b/src/transformers/models/vit_mae/modeling_vit_mae.py index e85d996f47b1..ad7fb8b8aa69 100755 --- a/src/transformers/models/vit_mae/modeling_vit_mae.py +++ b/src/transformers/models/vit_mae/modeling_vit_mae.py @@ -416,8 +416,24 @@ def __init__(self, config: ViTMAEConfig) -> None: self.attention_probs_dropout_prob = config.attention_probs_dropout_prob def forward( - self, hidden_states, head_mask: Optional[torch.Tensor] = None, output_attentions: bool = False + self, + hidden_states: torch.FloatTensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: + if output_attentions or head_mask is not None: + logger.warning_once( + "`ViTMAESdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " + "`output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but " + "specifying the manual implementation will be required from Transformers version v5.0.0 onwards. " + 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + head_mask=head_mask, + output_attentions=output_attentions, + ) + mixed_query_layer = self.query(hidden_states) key_layer = self.transpose_for_scores(self.key(hidden_states)) diff --git a/src/transformers/models/vit_msn/modeling_vit_msn.py b/src/transformers/models/vit_msn/modeling_vit_msn.py index c89370be5c0f..332701961b24 100644 --- a/src/transformers/models/vit_msn/modeling_vit_msn.py +++ b/src/transformers/models/vit_msn/modeling_vit_msn.py @@ -228,8 +228,24 @@ def __init__(self, config: ViTMSNConfig) -> None: self.attention_probs_dropout_prob = config.attention_probs_dropout_prob def forward( - self, hidden_states, head_mask: Optional[torch.Tensor] = None, output_attentions: bool = False + self, + hidden_states: torch.FloatTensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: + if output_attentions or head_mask is not None: + logger.warning_once( + "`ViTMSNSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " + "`output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but " + "specifying the manual implementation will be required from Transformers version v5.0.0 onwards. " + 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + head_mask=head_mask, + output_attentions=output_attentions, + ) + mixed_query_layer = self.query(hidden_states) key_layer = self.transpose_for_scores(self.key(hidden_states)) diff --git a/src/transformers/models/yolos/modeling_yolos.py b/src/transformers/models/yolos/modeling_yolos.py index 2acf48849abc..a161dc1153f3 100755 --- a/src/transformers/models/yolos/modeling_yolos.py +++ b/src/transformers/models/yolos/modeling_yolos.py @@ -313,8 +313,24 @@ def __init__(self, config: YolosConfig) -> None: self.attention_probs_dropout_prob = config.attention_probs_dropout_prob def forward( - self, hidden_states, head_mask: Optional[torch.Tensor] = None, output_attentions: bool = False + self, + hidden_states: torch.FloatTensor, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: + if output_attentions or head_mask is not None: + logger.warning_once( + "`YolosSdpaAttention` is used but `torch.nn.functional.scaled_dot_product_attention` does not support " + "`output_attentions=True` or `head_mask`. Falling back to the manual attention implementation, but " + "specifying the manual implementation will be required from Transformers version v5.0.0 onwards. " + 'This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + head_mask=head_mask, + output_attentions=output_attentions, + ) + mixed_query_layer = self.query(hidden_states) key_layer = self.transpose_for_scores(self.key(hidden_states)) From 364853785eb7097619e37f31ae318cdca1ef0787 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 1 Aug 2024 09:51:38 +0200 Subject: [PATCH 23/68] update --- src/transformers/modeling_utils.py | 12 +++----- .../modeling_encoder_decoder.py | 6 ++-- .../models/idefics2/modeling_idefics2.py | 16 ++-------- .../models/llava/modeling_llava.py | 1 - .../test_modeling_encoder_decoder.py | 30 +++++++++++-------- tests/models/idefics/test_modeling_idefics.py | 4 --- .../models/idefics2/test_modeling_idefics2.py | 10 +++---- .../test_modeling_vision_encoder_decoder.py | 30 +++++++++++-------- tests/test_modeling_common.py | 16 ++++++++-- 9 files changed, 64 insertions(+), 61 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index b13c86605b92..132fdb2849ee 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1523,6 +1523,7 @@ def _autoset_attn_implementation( # typical SDPA-dispatch path (i.e. if hard_check). Same goes for FA2. sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} if sub_configs: + attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): sub_model_cls = MODEL_MAPPING.get(type(sub_config), None) if sub_model_cls is not None: @@ -1532,8 +1533,6 @@ def _autoset_attn_implementation( sub_config._attn_implementation = requested_attn_implementation[key] else: sub_config._attn_implementation = requested_attn_implementation - - print(type(sub_config), requested_attn_implementation) sub_model_cls._autoset_attn_implementation( sub_config, use_flash_attention_2=use_flash_attention_2, @@ -1542,13 +1541,10 @@ def _autoset_attn_implementation( check_device_map=check_device_map, ) setattr(config, key, sub_config) + attn_implementation_per_subconfig[key] = sub_config._attn_implementation - # Set to eager to avoid hard_check errors that cls does not support sdpa/FA2, because generalist model's - # cls attr should be set to False by default. Sub-configs cls attr can still be set to True - # If any sub-config doesnt support requested attn, we'll catch that when dispatching the sub-config above - requested_attn_implementation = ( - None if requested_attn_implementation == "sdpa" else requested_attn_implementation - ) + # Set the general attn_implementation to a dict where keys are sub-configs + requested_attn_implementation = attn_implementation_per_subconfig if use_flash_attention_2: logger.warning_once( diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index db65f6e5250f..7d21c4cefe1f 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -209,12 +209,14 @@ def __init__( if encoder is None: from ..auto.modeling_auto import AutoModel - encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation) + encoder = AutoModel.from_config(config.encoder, attn_implementation=config.encoder._attn_implementation) if decoder is None: from ..auto.modeling_auto import AutoModelForCausalLM - decoder = AutoModelForCausalLM.from_config(config.decoder, attn_implementation=config._attn_implementation) + decoder = AutoModelForCausalLM.from_config( + config.decoder, attn_implementation=config.decoder._attn_implementation + ) self.encoder = encoder self.decoder = decoder diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 7e88687c0e7b..70a66095b087 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -1133,18 +1133,6 @@ def _autoset_attn_implementation( check_device_map=check_device_map, **kwargs, ) - # autoset-attn calls recursively all sub-configs (text-config, vision-config) - # and sets attn implementation if the config can be mapped bu auto-model - # Idefics2 vision config can't be mapped automatically so we set it manually here - # We can't set vision attn same as general attn, because the general attr will be sdpa if at - # least one sub-module (in this case LLM) supports sdpa, and we know vision/perceiver doesn't support yet - if hasattr(config, "vision_config"): - config.vision_config._attn_implementation = ( - config._attn_implementation if config._attn_implementation != "sdpa" else "eager" - ) - config.perceiver_config._attn_implementation = ( - config._attn_implementation if config._attn_implementation != "sdpa" else "eager" - ) return config @@ -1230,7 +1218,9 @@ def __init__(self, config: Idefics2Config): self.vision_model = Idefics2VisionTransformer(config.vision_config) self.connector = Idefics2Connector(config) - self.text_model = AutoModel.from_config(config.text_config, attn_implementation=config._attn_implementation) + self.text_model = AutoModel.from_config( + config.text_config, attn_implementation=config.text_config._attn_implementation + ) self.image_seq_len = config.perceiver_config.resampler_n_latents self.image_token_id = self.config.image_token_id diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 8022d23e958f..9c9bcf95dcb2 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -238,7 +238,6 @@ def __init__(self, config: LlavaConfig): config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 - print(config.vision_config._attn_implementation, config.text_config._attn_implementation) self.post_init() def get_input_embeddings(self): diff --git a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py index 9bb9982572c6..e973bc47f09c 100644 --- a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py +++ b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py @@ -723,23 +723,27 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - # We didn't skip test, so this model supports sdpa for sure in general config - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue( + model_sdpa.config._attn_implementation == {"encoder": encoder_attn, "decoder": decoder_attn} + ) self.assertTrue(model_sdpa.config.encoder._attn_implementation == encoder_attn) self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) - # Also test that nothing break if we request SDPA explicitly - # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible + # Also test that nothing break if we request SDPA explicitly, when both sub-parts support it. + # If the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA - # Checking error is out-of-scope of this test - model_sdpa_explicit = EncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) - model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) + if encoder_attn == "sdpa" and decoder_attn == "sdpa": + model_sdpa_explicit = EncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) + model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") - self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) - self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) + self.assertTrue( + model_sdpa_explicit.config._attn_implementation + == {"encoder": encoder_attn, "decoder": decoder_attn} + ) + self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) model_eager = EncoderDecoderModel.from_pretrained( tmpdirname, @@ -748,7 +752,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) self.assertTrue(model_eager.config.encoder._attn_implementation == "eager") self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index 1cf085b6ee66..2ddc7612208f 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -601,8 +601,6 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): perceiver_attn = "eager" vision_attn = "eager" - # We also know this model supports sdpa for sure in general config - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) @@ -615,7 +613,6 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa_explicit.config.perceiver_config._attn_implementation == perceiver_attn) self.assertTrue(model_sdpa_explicit.config.vision_config._attn_implementation == vision_attn) @@ -626,7 +623,6 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") self.assertTrue(model_eager.config.perceiver_config._attn_implementation == "eager") self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 9b7e4b50cedb..b85e6d351bea 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -356,8 +356,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): perceiver_attn = "eager" vision_attn = "eager" - # We didn't skip test, so this model supports sdpa for sure in general config - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa.config.text_config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) @@ -370,7 +369,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") + self.assertTrue(model_sdpa_explicit.config.text_config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) @@ -381,7 +380,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == "eager") self.assertTrue(model_sdpa.config.vision_config._attn_implementation == "eager") @@ -390,13 +389,14 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: raise ValueError("The eager model should not have SDPA attention layers") + print(model_sdpa) has_sdpa = False for name, submodule in model_sdpa.named_modules(): class_name = submodule.__class__.__name__ if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: has_sdpa = True break - if not has_sdpa and model_sdpa.config.model_type != "falcon": + if not has_sdpa: raise ValueError("The SDPA model should have SDPA attention layers") diff --git a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py index 2f341f3df3ca..69addc0e8ec5 100644 --- a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py +++ b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py @@ -433,23 +433,27 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - # We didn't skip test, so this model supports sdpa for sure in general config - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue( + model_sdpa.config._attn_implementation == {"encoder": encoder_attn, "decoder": decoder_attn} + ) self.assertTrue(model_sdpa.config.encoder._attn_implementation == encoder_attn) self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) - # Also test that nothing break if we request SDPA explicitly - # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible + # Also test that nothing break if we request SDPA explicitly, when both sub-parts support it. + # If the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA - # Checking error is out-of-scope of this test - model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) - model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) + if encoder_attn == "sdpa" and decoder_attn == "sdpa": + model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) + model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") - self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) - self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) + self.assertTrue( + model_sdpa_explicit.config._attn_implementation + == {"encoder": encoder_attn, "decoder": decoder_attn} + ) + self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) model_eager = VisionEncoderDecoderModel.from_pretrained( tmpdirname, @@ -458,7 +462,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) self.assertTrue(model_eager.config.encoder._attn_implementation == "eager") self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 0e863f2d1ec3..12abfaf0963d 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3777,7 +3777,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") if self.is_multimodal: vision_supports_sdpa = ( model.image_tower._supports_sdpa @@ -3785,7 +3784,15 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): else model.vision_tower._supports_sdpa ) vision_attn = "sdpa" if vision_supports_sdpa else "eager" + text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.config.text_config._attn_implementation == text_attn) + self.assertTrue( + model_sdpa.config._attn_implementation + == {"text_config": text_attn, "vision_config": vision_attn} + ) + else: + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") model_eager = model_class.from_pretrained( tmpdirname, @@ -3794,9 +3801,14 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") if self.is_multimodal: self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") + self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} + ) + else: + self.assertTrue(model_eager.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ From ca95feefbe4e0907979e94a2f99b31c5bba9029e Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 1 Aug 2024 09:53:03 +0200 Subject: [PATCH 24/68] muicgen --- .../models/musicgen/configuration_musicgen.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/transformers/models/musicgen/configuration_musicgen.py b/src/transformers/models/musicgen/configuration_musicgen.py index ef2e0244c140..0d282355defa 100644 --- a/src/transformers/models/musicgen/configuration_musicgen.py +++ b/src/transformers/models/musicgen/configuration_musicgen.py @@ -236,20 +236,3 @@ def from_sub_models_config( # This is a property because you might want to change the codec model on the fly def sampling_rate(self): return self.audio_encoder.sampling_rate - - @property - def _attn_implementation(self): - # This property is made private for now (as it cannot be changed and a PreTrainedModel.use_attn_implementation method needs to be implemented.) - if hasattr(self, "_attn_implementation_internal"): - if self._attn_implementation_internal is None: - # `config.attn_implementation` should never be None, for backward compatibility. - return "eager" - else: - return self._attn_implementation_internal - else: - return "eager" - - @_attn_implementation.setter - def _attn_implementation(self, value): - self._attn_implementation_internal = value - self.decoder._attn_implementation = value From 79cae6d521366d049a282dd6bb454683c43399a5 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 1 Aug 2024 10:00:33 +0200 Subject: [PATCH 25/68] no prints --- src/transformers/models/vipllava/modeling_vipllava.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 2108b1a73ba5..58ebf2db2dea 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -243,7 +243,6 @@ def __init__(self, config: VipLlavaConfig): config.text_config, attn_implementation=config.text_config._attn_implementation ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 - print(config.vision_config._attn_implementation, config.text_config._attn_implementation) self.post_init() def get_input_embeddings(self): From 378274b02e30e088a0701f6232908beba5a3678c Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 1 Aug 2024 10:34:46 +0200 Subject: [PATCH 26/68] forgot speech enc-dec and clip --- src/transformers/models/clip/modeling_clip.py | 16 +++++++++++----- .../modeling_speech_encoder_decoder.py | 6 ++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index ee85fe312587..81ff2ca51f36 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -1137,10 +1137,14 @@ def __init__(self, config: CLIPConfig): self.text_embed_dim = text_config.hidden_size self.vision_embed_dim = vision_config.hidden_size - text_model = CLIPTextModel._from_config(text_config, attn_implementation=config._attn_implementation) + text_model = CLIPTextModel._from_config( + text_config, attn_implementation=config.text_config._attn_implementation + ) self.text_model = text_model.text_model - vision_model = CLIPVisionModel._from_config(vision_config, attn_implementation=config._attn_implementation) + vision_model = CLIPVisionModel._from_config( + vision_config, attn_implementation=config.vision_config._attn_implementation + ) self.vision_model = vision_model.vision_model self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False) @@ -1356,7 +1360,7 @@ class CLIPTextModelWithProjection(CLIPPreTrainedModel): def __init__(self, config: CLIPTextConfig): super().__init__(config) - text_model = CLIPTextModel._from_config(config, attn_implementation=config._attn_implementation) + text_model = CLIPTextModel._from_config(config, attn_implementation=config.text_config._attn_implementation) self.text_model = text_model.text_model self.text_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) @@ -1437,7 +1441,9 @@ class CLIPVisionModelWithProjection(CLIPPreTrainedModel): def __init__(self, config: CLIPVisionConfig): super().__init__(config) - vision_model = CLIPVisionModel._from_config(config, attn_implementation=config._attn_implementation) + vision_model = CLIPVisionModel._from_config( + config, attn_implementation=config.vision_config._attn_implementation + ) self.vision_model = vision_model.vision_model self.visual_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) @@ -1518,7 +1524,7 @@ def __init__(self, config: CLIPConfig) -> None: self.num_labels = config.num_labels vision_model = CLIPVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config.vision_config._attn_implementation ) self.vision_model = vision_model.vision_model diff --git a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py index c2f5dd025909..3eece905f034 100644 --- a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py +++ b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py @@ -212,10 +212,12 @@ def __init__( super().__init__(config) if encoder is None: - encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation) + encoder = AutoModel.from_config(config.encoder, attn_implementation=config.encoder._attn_implementation) if decoder is None: - decoder = AutoModelForCausalLM.from_config(config.decoder, attn_implementation=config._attn_implementation) + decoder = AutoModelForCausalLM.from_config( + config.decoder, attn_implementation=config.decoder._attn_implementation + ) self.encoder = encoder self.decoder = decoder From a772ff515e7e537cf3557640ff0acb97f470dce8 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 1 Aug 2024 11:05:02 +0200 Subject: [PATCH 27/68] how many composite models we have? --- src/transformers/models/clip/modeling_clip.py | 6 ++---- .../modeling_vision_text_dual_encoder.py | 6 ++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 81ff2ca51f36..63bbf46a30ca 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -1360,7 +1360,7 @@ class CLIPTextModelWithProjection(CLIPPreTrainedModel): def __init__(self, config: CLIPTextConfig): super().__init__(config) - text_model = CLIPTextModel._from_config(config, attn_implementation=config.text_config._attn_implementation) + text_model = CLIPTextModel._from_config(config, attn_implementation=config._attn_implementation) self.text_model = text_model.text_model self.text_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) @@ -1441,9 +1441,7 @@ class CLIPVisionModelWithProjection(CLIPPreTrainedModel): def __init__(self, config: CLIPVisionConfig): super().__init__(config) - vision_model = CLIPVisionModel._from_config( - config, attn_implementation=config.vision_config._attn_implementation - ) + vision_model = CLIPVisionModel._from_config(config, attn_implementation=config._attn_implementation) self.vision_model = vision_model.vision_model self.visual_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 5b90faa8862c..383f4f75ccf3 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -185,11 +185,13 @@ def __init__( vision_model = CLIPVisionModel(config.vision_config) else: vision_model = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config.vision_config._attn_implementation ) if text_model is None: - text_model = AutoModel.from_config(config.text_config, attn_implementation=config._attn_implementation) + text_model = AutoModel.from_config( + config.text_config, attn_implementation=config.text_config._attn_implementation + ) self.vision_model = vision_model self.text_model = text_model From 3e6787c73e4303038c4a357209eee81f530c38d5 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 1 Aug 2024 11:50:06 +0200 Subject: [PATCH 28/68] musicgen meelody is same as mudicgen --- .../configuration_musicgen_melody.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/transformers/models/musicgen_melody/configuration_musicgen_melody.py b/src/transformers/models/musicgen_melody/configuration_musicgen_melody.py index b29187facb3d..8a77cea02522 100644 --- a/src/transformers/models/musicgen_melody/configuration_musicgen_melody.py +++ b/src/transformers/models/musicgen_melody/configuration_musicgen_melody.py @@ -250,20 +250,3 @@ def from_sub_models_config( # This is a property because you might want to change the codec model on the fly def sampling_rate(self): return self.audio_encoder.sampling_rate - - @property - def _attn_implementation(self): - # This property is made private for now (as it cannot be changed and a PreTrainedModel.use_attn_implementation method needs to be implemented.) - if hasattr(self, "_attn_implementation_internal"): - if self._attn_implementation_internal is None: - # `config.attn_implementation` should never be None, for backward compatibility. - return "eager" - else: - return self._attn_implementation_internal - else: - return "eager" - - @_attn_implementation.setter - def _attn_implementation(self, value): - self._attn_implementation_internal = value - self.decoder._attn_implementation = value From 00b206564aa29563b46bfce0511de4a4f1eefc8c Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 1 Aug 2024 12:50:57 +0200 Subject: [PATCH 29/68] +siglip --- src/transformers/models/siglip/modeling_siglip.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 797a8fa0c0ef..fb6cd4355f32 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -1217,8 +1217,12 @@ def __init__(self, config: SiglipConfig): vision_config = config.vision_config # First, initialize the text and vision models with proper attention implementation - text_model = SiglipTextModel._from_config(text_config, attn_implementation=config._attn_implementation) - vision_model = SiglipVisionModel._from_config(vision_config, attn_implementation=config._attn_implementation) + text_model = SiglipTextModel._from_config( + text_config, attn_implementation=config.text_config._attn_implementation + ) + vision_model = SiglipVisionModel._from_config( + vision_config, attn_implementation=config.vision_config._attn_implementation + ) # Second, get the text and vision submodules (for backward compatibility) self.text_model = text_model.text_model @@ -1454,7 +1458,7 @@ def __init__(self, config: SiglipConfig) -> None: # Create the vision model with proper attention # and take only vision_model submodule (for backward compatibility) vision_model = SiglipVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config.vision_config._attn_implementation ) self.vision_model = vision_model.vision_model From 5cdfbfb151bccf25140e8ef8cc8a6dde688612c9 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 2 Aug 2024 08:47:46 +0200 Subject: [PATCH 30/68] fix tests + add some more --- src/transformers/modeling_utils.py | 26 ++++++------------- .../models/auto/configuration_auto.py | 12 +++++++++ src/transformers/models/auto/modeling_auto.py | 4 +++ .../models/blip_2/modeling_blip_2.py | 6 +++-- src/transformers/models/clip/modeling_clip.py | 1 + .../modeling_encoder_decoder.py | 1 + .../models/idefics/modeling_idefics.py | 1 + .../models/idefics2/modeling_idefics2.py | 1 + .../instructblip/modeling_instructblip.py | 2 ++ .../modeling_instructblipvideo.py | 2 ++ .../models/llava/modeling_llava.py | 1 + .../models/llava_next/modeling_llava_next.py | 1 + .../modeling_llava_next_video.py | 1 + .../models/musicgen/modeling_musicgen.py | 2 ++ .../modeling_musicgen_melody.py | 2 ++ .../models/paligemma/modeling_paligemma.py | 1 + .../models/siglip/modeling_siglip.py | 2 ++ .../video_llava/modeling_video_llava.py | 1 + .../models/vipllava/modeling_vipllava.py | 1 + .../modeling_vision_encoder_decoder.py | 1 + .../modeling_vision_text_dual_encoder.py | 1 + tests/models/clip/test_modeling_clip.py | 23 ++++++++++++++-- .../models/musicgen/test_modeling_musicgen.py | 23 ++++++++++++++-- .../test_modeling_musicgen_melody.py | 23 ++++++++++++++-- tests/models/siglip/test_modeling_siglip.py | 23 ++++++++++++++-- utils/check_copies.py | 2 ++ utils/check_repo.py | 3 --- utils/check_table.py | 10 ++++++- 28 files changed, 145 insertions(+), 32 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 132fdb2849ee..bd28f513d701 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1331,6 +1331,9 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix # SDPA support _supports_sdpa = False + # Composite models consisting of several PretrainedModels + _is_composite = False + # Has support for a `Cache` instance as `past_key_values`? Does it support a `StaticCache`? _supports_cache_class = False _supports_static_cache = False @@ -1544,7 +1547,9 @@ def _autoset_attn_implementation( attn_implementation_per_subconfig[key] = sub_config._attn_implementation # Set the general attn_implementation to a dict where keys are sub-configs - requested_attn_implementation = attn_implementation_per_subconfig + requested_attn_implementation = ( + attn_implementation_per_subconfig if cls._is_composite else requested_attn_implementation + ) if use_flash_attention_2: logger.warning_once( @@ -1647,8 +1652,7 @@ def _check_and_enable_flash_attn_2( """ # VLM/Encoder-Decoder etc. have to follow the sdpa attr of its sub-configs - sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} - if not cls._supports_flash_attn_2 and not sub_configs: + if not cls._supports_flash_attn_2: raise ValueError( f"{cls.__name__} does not support Flash Attention 2.0 yet. Please request to add support where" f" the model is hosted, on its model hub page: https://huggingface.co/{config._name_or_path}/discussions/new" @@ -1721,15 +1725,7 @@ def _check_and_enable_flash_attn_2( "initialise the model on a GPU by passing a device_map that contains only GPU devices as keys." ) if not hard_check_only: - if sub_configs: - sub_config_attentions = {sub_config._attn_implementation for key, sub_config in sub_configs.items()} - config._attn_implementation = ( - "flash_attention_2" - if "flash_attention_2" in sub_config_attentions - else config._attn_implementation - ) - else: - config._attn_implementation = "flash_attention_2" + config._attn_implementation = "flash_attention_2" return config @classmethod @@ -1751,12 +1747,6 @@ def _check_and_enable_sdpa(cls, config, hard_check_only: bool = False) -> Pretra "PyTorch SDPA requirements in Transformers are not met. Please install torch>=2.1.1." ) - # VLM/Encoder-Decoder etc. have to follow the sdpa attr of its sub-configs - sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} - if sub_configs: - sub_config_attentions = {sub_config._attn_implementation for key, sub_config in sub_configs.items()} - config._attn_implementation = "sdpa" if "sdpa" in sub_config_attentions else config._attn_implementation - if not is_torch_sdpa_available() or not cls._supports_sdpa: return config diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 512c1eaaf5e0..5734499f644f 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -60,6 +60,7 @@ ("chinese_clip_vision_model", "ChineseCLIPVisionConfig"), ("clap", "ClapConfig"), ("clip", "CLIPConfig"), + ("clip_text_model", "CLIPTextConfig"), ("clip_vision_model", "CLIPVisionConfig"), ("clipseg", "CLIPSegConfig"), ("clvp", "ClvpConfig"), @@ -174,7 +175,9 @@ ("mra", "MraConfig"), ("mt5", "MT5Config"), ("musicgen", "MusicgenConfig"), + ("musicgen_decoder", "MusicgenDecoderConfig"), ("musicgen_melody", "MusicgenMelodyConfig"), + ("musicgen_melody_decoder", "MusicgenMelodyDecoderConfig"), ("mvp", "MvpConfig"), ("nat", "NatConfig"), ("nezha", "NezhaConfig"), @@ -230,6 +233,7 @@ ("sew", "SEWConfig"), ("sew-d", "SEWDConfig"), ("siglip", "SiglipConfig"), + ("siglip_text_model", "SiglipTextConfig"), ("siglip_vision_model", "SiglipVisionConfig"), ("speech-encoder-decoder", "SpeechEncoderDecoderConfig"), ("speech_to_text", "Speech2TextConfig"), @@ -335,6 +339,7 @@ ("chinese_clip_vision_model", "ChineseCLIPVisionModel"), ("clap", "CLAP"), ("clip", "CLIP"), + ("clip_text_model", "CLIPTextModel"), ("clip_vision_model", "CLIPVisionModel"), ("clipseg", "CLIPSeg"), ("clvp", "CLVP"), @@ -466,7 +471,9 @@ ("mra", "MRA"), ("mt5", "MT5"), ("musicgen", "MusicGen"), + ("musicgen_decoder", "MusicGenDecoder"), ("musicgen_melody", "MusicGen Melody"), + ("musicgen_melody_decoder", "MusicGen Melody Decoder"), ("mvp", "MVP"), ("nat", "NAT"), ("nezha", "Nezha"), @@ -524,6 +531,7 @@ ("sew", "SEW"), ("sew-d", "SEW-D"), ("siglip", "SigLIP"), + ("siglip_text_model", "SiglipTextModel"), ("siglip_vision_model", "SiglipVisionModel"), ("speech-encoder-decoder", "Speech Encoder decoder"), ("speech_to_text", "Speech2Text"), @@ -636,8 +644,12 @@ ("donut-swin", "donut"), ("kosmos-2", "kosmos2"), ("maskformer-swin", "maskformer"), + ("musicgen_decoder", "musicgen"), + ("musicgen_melody_decoder", "musicgen_melody"), ("xclip", "x_clip"), + ("clip_text_model", "clip"), ("clip_vision_model", "clip"), + ("siglip_text_model", "siglip"), ("siglip_vision_model", "siglip"), ("chinese_clip_vision_model", "chinese_clip"), ("rt_detr_resnet", "rt_detr"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index d096abf43426..b2fbf35da241 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -60,6 +60,7 @@ ("chinese_clip_vision_model", "ChineseCLIPVisionModel"), ("clap", "ClapModel"), ("clip", "CLIPModel"), + ("clip_text_model", "CLIPTextModel"), ("clip_vision_model", "CLIPVisionModel"), ("clipseg", "CLIPSegModel"), ("clvp", "ClvpModelForConditionalGeneration"), @@ -166,7 +167,9 @@ ("mra", "MraModel"), ("mt5", "MT5Model"), ("musicgen", "MusicgenModel"), + ("musicgen_decoder", "MusicgenModel"), ("musicgen_melody", "MusicgenMelodyModel"), + ("musicgen_melody_decoder", "MusicgenMelodyModel"), ("mvp", "MvpModel"), ("nat", "NatModel"), ("nezha", "NezhaModel"), @@ -215,6 +218,7 @@ ("sew", "SEWModel"), ("sew-d", "SEWDModel"), ("siglip", "SiglipModel"), + ("siglip_text_model", "SiglipTextModel"), ("siglip_vision_model", "SiglipVisionModel"), ("speech_to_text", "Speech2TextModel"), ("speecht5", "SpeechT5Model"), diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 8028cd1e8777..e335013e52df 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -304,6 +304,8 @@ class Blip2PreTrainedModel(PreTrainedModel): config_class = Blip2Config base_model_prefix = "blip" supports_gradient_checkpointing = True + _is_composite = True + _no_split_modules = ["Blip2Attention", "T5Block", "OPTDecoderLayer"] _skip_keys_device_placement = "past_key_values" _keep_in_fp32_modules = ["wo"] @@ -1602,11 +1604,11 @@ def __init__(self, config: Blip2Config): self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config.text_config._attn_implementation ) # Update _tied_weights_keys using the base model used. diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 63bbf46a30ca..cf784a2d7686 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -577,6 +577,7 @@ class CLIPPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _supports_sdpa = True _supports_flash_attn_2 = True + _is_composite = True def _init_weights(self, module): """Initialize the weights""" diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index 7d21c4cefe1f..2e3681318d24 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -179,6 +179,7 @@ class EncoderDecoderModel(PreTrainedModel): main_input_name = "input_ids" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False + _is_composite = True def __init__( self, diff --git a/src/transformers/models/idefics/modeling_idefics.py b/src/transformers/models/idefics/modeling_idefics.py index 6d6582598609..80c51f9e269b 100644 --- a/src/transformers/models/idefics/modeling_idefics.py +++ b/src/transformers/models/idefics/modeling_idefics.py @@ -941,6 +941,7 @@ class IdeficsPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["IdeficsDecoderLayer", "IdeficsGatedCrossAttentionLayer"] _supports_sdpa = True + _is_composite = True def _init_weights(self, module): # important: this ported version of Idefics isn't meant for training from scratch - only diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 70a66095b087..3949f13b0864 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -1092,6 +1092,7 @@ class Idefics2PreTrainedModel(PreTrainedModel): _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True _supports_cache_class = True + _is_composite = True def _init_weights(self, module): std = ( diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 40135148ed3b..4ba0fc1bf1b8 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -306,6 +306,8 @@ class InstructBlipPreTrainedModel(PreTrainedModel): config_class = InstructBlipConfig base_model_prefix = "blip" supports_gradient_checkpointing = True + _is_composite = True + _no_split_modules = [ "InstructBlipQFormerEmbeddings", "InstructBlipAttention", diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 62330de6042d..2254eff9d21c 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -314,6 +314,8 @@ class InstructBlipVideoPreTrainedModel(PreTrainedModel): config_class = InstructBlipVideoConfig base_model_prefix = "blip" supports_gradient_checkpointing = True + _is_composite = True + _no_split_modules = [ "InstructBlipVideoQFormerEmbeddings", "InstructBlipVideoAttention", diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 9c9bcf95dcb2..57de61b731c7 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -126,6 +126,7 @@ class LlavaPreTrainedModel(PreTrainedModel): _no_split_modules = ["LlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _is_composite = True def _init_weights(self, module): # important: this ported version of Llava isn't meant for training from scratch - only diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 0687c4c1d534..b7ad164d123d 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -232,6 +232,7 @@ class LlavaNextPreTrainedModel(PreTrainedModel): _no_split_modules = ["LlavaNextVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _is_composite = True def _init_weights(self, module): # important: this ported version of LlavaNext isn't meant for training from scratch - only diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 11247f037069..b275e1fd200e 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -272,6 +272,7 @@ class LlavaNextVideoPreTrainedModel(PreTrainedModel): _no_split_modules = ["LlavaNextVideoVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _is_composite = True def _init_weights(self, module): # important: this ported version of LlavaNextVideo isn't meant for training from scratch - only diff --git a/src/transformers/models/musicgen/modeling_musicgen.py b/src/transformers/models/musicgen/modeling_musicgen.py index b0e456db8add..131195c847bb 100644 --- a/src/transformers/models/musicgen/modeling_musicgen.py +++ b/src/transformers/models/musicgen/modeling_musicgen.py @@ -704,6 +704,7 @@ class MusicgenPreTrainedModel(PreTrainedModel): _no_split_modules = ["MusicgenDecoderLayer", "MusicgenAttention"] _supports_flash_attn_2 = True _supports_sdpa = True + _is_composite = True def _init_weights(self, module): std = self.config.initializer_factor @@ -1673,6 +1674,7 @@ class MusicgenForConditionalGeneration(PreTrainedModel): supports_gradient_checkpointing = True _supports_flash_attn_2 = True _supports_sdpa = True + _is_composite = True def __init__( self, diff --git a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py index eafb7baad8f7..57fa643388d3 100644 --- a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py +++ b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py @@ -663,6 +663,7 @@ class MusicgenMelodyPreTrainedModel(PreTrainedModel): _no_split_modules = ["MusicgenMelodyDecoderLayer", "MusicgenMelodyAttention"] _supports_flash_attn_2 = True _supports_sdpa = True + _is_composite = True def _init_weights(self, module): std = self.config.initializer_factor @@ -1599,6 +1600,7 @@ class MusicgenMelodyForConditionalGeneration(PreTrainedModel): supports_gradient_checkpointing = True _supports_flash_attn_2 = True _supports_sdpa = True + _is_composite = True def __init__( self, diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 2fffb36f1cd3..57fa9c1e3b71 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -126,6 +126,7 @@ class PaliGemmaPreTrainedModel(PreTrainedModel): _no_split_modules = ["PaliGemmaMultiModalProjector"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = False + _is_composite = True def _init_weights(self, module): # important: this ported version of PaliGemmaisn't meant for training from scratch - only diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index fb6cd4355f32..9cb94935ba7d 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -668,6 +668,8 @@ class SiglipPreTrainedModel(PreTrainedModel): config_class = SiglipConfig base_model_prefix = "siglip" supports_gradient_checkpointing = True + _is_composite = True + _no_split_modules = [ "SiglipTextEmbeddings", "SiglipEncoderLayer", diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index aea575738316..8df37577fe75 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -126,6 +126,7 @@ class VideoLlavaPreTrainedModel(PreTrainedModel): _no_split_modules = ["VideoLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _is_composite = True def _init_weights(self, module): std = ( diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 58ebf2db2dea..88a3690a3a28 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -135,6 +135,7 @@ class VipLlavaPreTrainedModel(PreTrainedModel): _no_split_modules = ["VipLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _is_composite = True def _init_weights(self, module): # important: this ported version of VipLlava isn't meant for training from scratch - only diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py index ced1a48a45a4..1910c9a8912d 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py @@ -160,6 +160,7 @@ class VisionEncoderDecoderModel(PreTrainedModel): main_input_name = "pixel_values" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False + _is_composite = True def __init__( self, diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 383f4f75ccf3..f54e94b53358 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -161,6 +161,7 @@ def clip_loss(similarity: torch.Tensor) -> torch.Tensor: class VisionTextDualEncoderModel(PreTrainedModel): config_class = VisionTextDualEncoderConfig base_model_prefix = "vision_text_dual_encoder" + _is_composite = True def __init__( self, diff --git a/tests/models/clip/test_modeling_clip.py b/tests/models/clip/test_modeling_clip.py index 3b6994428088..ba85c7bc4468 100644 --- a/tests/models/clip/test_modeling_clip.py +++ b/tests/models/clip/test_modeling_clip.py @@ -196,6 +196,7 @@ def test_eager_matches_sdpa_inference( torch_dtype: str, use_attention_mask_options: Tuple[Optional[str], ...] = (None, "left", "right"), logit_keys: Tuple[str, ...] = ("logits_per_image", "logits_per_text", "image_embeds", "text_embeds"), + is_composite: bool = True, ): if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") @@ -252,8 +253,24 @@ def get_mean_reldiff(msg, current_case, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - self.assertTrue(model_eager.config._attn_implementation == "eager") + if not is_composite: + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_eager.config._attn_implementation == "eager") + else: + vision_attn = text_attn = ( + "sdpa" if model._supports_sdpa else "eager" + ) # sigLip has one shared cls attr for all models + self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.config.text_config._attn_implementation == text_attn) + self.assertTrue( + model_sdpa.config._attn_implementation == {"text_config": text_attn, "vision_config": vision_attn} + ) + + self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") + self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} + ) for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ @@ -459,6 +476,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("last_hidden_state", "pooler_output", "image_embeds"), use_attention_mask_options=(None,), + is_composite=False, ) @@ -637,6 +655,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("last_hidden_state", "pooler_output", "text_embeds"), use_attention_mask_options=(None, "right"), # "left" is not supported for text model + is_composite=False, ) @require_torch_sdpa diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 6ad93d3268ec..c559d5317332 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -2004,7 +2004,20 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" + text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + self.assertTrue(model_sdpa.config.audio_encoder._attn_implementation == audio_encoder_attn) + self.assertTrue(model_sdpa.config.text_encoder._attn_implementation == text_encoder_attn) + self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + self.assertTrue( + model_sdpa.config._attn_implementation + == { + "audio_encoder": audio_encoder_attn, + "text_encoder": text_encoder_attn, + "decoder": decoder_attn, + } + ) model_eager = model_class.from_pretrained( tmpdirname, @@ -2013,7 +2026,13 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config.audio_encoder._attn_implementation == "eager") + self.assertTrue(model_eager.config.text_encoder._attn_implementation == "eager") + self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation + == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} + ) for name, submodule in model_eager.named_modules(): if "SdpaAttention" in submodule.__class__.__name__: diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index f8486f5653c7..b65dec4f43c4 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -1983,7 +1983,20 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" + text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + self.assertTrue(model_sdpa.config.audio_encoder._attn_implementation == audio_encoder_attn) + self.assertTrue(model_sdpa.config.text_encoder._attn_implementation == text_encoder_attn) + self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + self.assertTrue( + model_sdpa.config._attn_implementation + == { + "audio_encoder": audio_encoder_attn, + "text_encoder": text_encoder_attn, + "decoder": decoder_attn, + } + ) model_eager = model_class.from_pretrained( tmpdirname, @@ -1992,7 +2005,13 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.config.audio_encoder._attn_implementation == "eager") + self.assertTrue(model_eager.config.text_encoder._attn_implementation == "eager") + self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation + == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} + ) for name, submodule in model_eager.named_modules(): if "SdpaAttention" in submodule.__class__.__name__: diff --git a/tests/models/siglip/test_modeling_siglip.py b/tests/models/siglip/test_modeling_siglip.py index 9d1e3109b313..480994b84092 100644 --- a/tests/models/siglip/test_modeling_siglip.py +++ b/tests/models/siglip/test_modeling_siglip.py @@ -76,6 +76,7 @@ def test_eager_matches_sdpa_inference( torch_dtype: str, use_attention_mask_options: Tuple[bool, ...] = (True, False), logit_keys: Tuple[str, ...] = ("logits_per_image", "logits_per_text", "image_embeds", "text_embeds"), + is_composite: bool = True, ): if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") @@ -132,8 +133,24 @@ def get_mean_reldiff(msg, current_case, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - self.assertTrue(model_eager.config._attn_implementation == "eager") + if not is_composite: + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_eager.config._attn_implementation == "eager") + else: + vision_attn = text_attn = ( + "sdpa" if model._supports_sdpa else "eager" + ) # sigLip has one shared cls attr for all models + self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.config.text_config._attn_implementation == text_attn) + self.assertTrue( + model_sdpa.config._attn_implementation == {"text_config": text_attn, "vision_config": vision_attn} + ) + + self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") + self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} + ) for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ @@ -398,6 +415,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("pooler_output", "last_hidden_state"), use_attention_mask_options=(False,), + is_composite=False, ) @@ -560,6 +578,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("pooler_output", "last_hidden_state"), use_attention_mask_options=(False, True), + is_composite=False, ) diff --git a/utils/check_copies.py b/utils/check_copies.py index 4bb5c6fef4ee..77da83a51787 100644 --- a/utils/check_copies.py +++ b/utils/check_copies.py @@ -1086,7 +1086,9 @@ def _find_text_in_file(filename: str, start_prompt: str, end_prompt: str) -> Tup "Vision Encoder decoder", "VisionTextDualEncoder", "CLIPVisionModel", + "CLIPTextModel", "SiglipVisionModel", + "SiglipTextModel", "ChineseCLIPVisionModel", ] diff --git a/utils/check_repo.py b/utils/check_repo.py index 293089ccb662..26e40112bf89 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -217,7 +217,6 @@ "BeitForMaskedImageModeling", "ChineseCLIPTextModel", "ChineseCLIPVisionModel", - "CLIPTextModel", "CLIPTextModelWithProjection", "CLIPVisionModelWithProjection", "ClvpForCausalLM", @@ -318,8 +317,6 @@ "SeamlessM4Tv2CodeHifiGan", "SeamlessM4Tv2ForSpeechToSpeech", # no auto class for speech-to-speech "SegGptForImageSegmentation", - "SiglipVisionModel", - "SiglipTextModel", "ChameleonVQVAE", # no autoclass for VQ-VAE models ] diff --git a/utils/check_table.py b/utils/check_table.py index 0866f6bf61ba..f62d6ce08a24 100644 --- a/utils/check_table.py +++ b/utils/check_table.py @@ -173,7 +173,15 @@ def _center_text(text: str, width: int) -> str: "XLS-R": "Wav2Vec2", "XLSR-Wav2Vec2": "Wav2Vec2", } -MODEL_NAMES_TO_IGNORE = ["CLIPVisionModel", "SiglipVisionModel", "ChineseCLIPVisionModel"] +MODEL_NAMES_TO_IGNORE = [ + "CLIPVisionModel", + "CLIPTextModel", + "SiglipVisionModel", + "SiglipTextModel", + "ChineseCLIPVisionModel", + "MusicGenDecoder", + "MusicGen Melody Decoder", +] def get_model_table_from_auto_modules() -> str: From 723f27dbe1c69332ea3508c643eb53271cabd088 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 2 Aug 2024 08:50:39 +0200 Subject: [PATCH 31/68] remove idefics custom overriden code --- .../models/idefics/modeling_idefics.py | 12 --------- .../models/idefics2/modeling_idefics2.py | 25 +------------------ 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/transformers/models/idefics/modeling_idefics.py b/src/transformers/models/idefics/modeling_idefics.py index 80c51f9e269b..9204a039db66 100644 --- a/src/transformers/models/idefics/modeling_idefics.py +++ b/src/transformers/models/idefics/modeling_idefics.py @@ -957,18 +957,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - # Adapted from transformers.modeling_utils.PreTrainedModel._check_and_enable_sdpa - @classmethod - def _check_and_enable_sdpa(cls, config, hard_check_only: bool = False) -> PretrainedConfig: - # We remove the checks on `is_torch_sdpa_available()` and `cls._supports_sdpa` as Falcon supports SDPA from torch==2.0.0 (no requirement on 2.1). - _is_bettertransformer = getattr(cls, "use_bettertransformer", False) - if _is_bettertransformer: - return config - - if not hard_check_only: - config._attn_implementation = "sdpa" - return config - LLAMA_INPUTS_DOCSTRING = r""" Args: diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 3949f13b0864..4284a6114e81 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -16,7 +16,7 @@ import math from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union import torch import torch.utils.checkpoint @@ -1113,29 +1113,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @classmethod - def _autoset_attn_implementation( - cls, - config, - use_flash_attention_2: bool = False, - torch_dtype: Optional[torch.dtype] = None, - device_map: Optional[Union[str, Dict[str, int]]] = None, - check_device_map: bool = True, - **kwargs, - ): - """ - Overrides the method in `PreTrainedModel` to update the vision config with the correct attention implementation - """ - config = super()._autoset_attn_implementation( - config=config, - use_flash_attention_2=use_flash_attention_2, - torch_dtype=torch_dtype, - device_map=device_map, - check_device_map=check_device_map, - **kwargs, - ) - return config - IDEFICS2_INPUTS_DOCSTRING = r""" Args: From 198c60c654cafeae80373899d3f422e41d7fcda3 Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 7 Aug 2024 08:38:45 +0200 Subject: [PATCH 32/68] make idefics2 automappable --- src/transformers/__init__.py | 8 +- .../models/auto/configuration_auto.py | 6 + src/transformers/models/auto/modeling_auto.py | 2 + src/transformers/models/idefics2/__init__.py | 8 +- .../models/idefics2/configuration_idefics2.py | 2 +- .../models/idefics2/modeling_idefics2.py | 169 +++++++++++------- src/transformers/utils/dummy_pt_objects.py | 14 ++ .../models/idefics2/test_modeling_idefics2.py | 23 +-- utils/check_table.py | 2 + 9 files changed, 149 insertions(+), 85 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 9108367f35b3..f24ee2cf5d96 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -472,7 +472,7 @@ "models.hubert": ["HubertConfig"], "models.ibert": ["IBertConfig"], "models.idefics": ["IdeficsConfig"], - "models.idefics2": ["Idefics2Config"], + "models.idefics2": ["Idefics2Config", "Idefics2PerceiverConfig", "Idefics2VisionConfig"], "models.imagegpt": ["ImageGPTConfig"], "models.informer": ["InformerConfig"], "models.instructblip": [ @@ -2344,8 +2344,10 @@ [ "Idefics2ForConditionalGeneration", "Idefics2Model", + "Idefics2PerceiverResampler", "Idefics2PreTrainedModel", "Idefics2Processor", + "Idefics2VisionTransformer", ] ) _import_structure["models.imagegpt"].extend( @@ -5152,7 +5154,7 @@ from .models.idefics import ( IdeficsConfig, ) - from .models.idefics2 import Idefics2Config + from .models.idefics2 import Idefics2Config, Idefics2PerceiverConfig, Idefics2VisionConfig from .models.imagegpt import ImageGPTConfig from .models.informer import InformerConfig from .models.instructblip import ( @@ -6871,8 +6873,10 @@ from .models.idefics2 import ( Idefics2ForConditionalGeneration, Idefics2Model, + Idefics2PerceiverResampler, Idefics2PreTrainedModel, Idefics2Processor, + Idefics2VisionTransformer, ) from .models.imagegpt import ( ImageGPTForCausalImageModeling, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 5734499f644f..5144fe795aad 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -129,6 +129,8 @@ ("ibert", "IBertConfig"), ("idefics", "IdeficsConfig"), ("idefics2", "Idefics2Config"), + ("idefics2_perceiver_model", "Idefics2PerceiverConfig"), + ("idefics2_vision_model", "Idefics2VisionConfig"), ("imagegpt", "ImageGPTConfig"), ("informer", "InformerConfig"), ("instructblip", "InstructBlipConfig"), @@ -416,6 +418,8 @@ ("ibert", "I-BERT"), ("idefics", "IDEFICS"), ("idefics2", "Idefics2"), + ("idefics2_perceiver_model", "Idefics2PerceiverResampler"), + ("idefics2_vision_model", "Idefics2VisionTransformer"), ("imagegpt", "ImageGPT"), ("informer", "Informer"), ("instructblip", "InstructBLIP"), @@ -653,6 +657,8 @@ ("siglip_vision_model", "siglip"), ("chinese_clip_vision_model", "chinese_clip"), ("rt_detr_resnet", "rt_detr"), + ("idefics2_vision_model", "idefics2"), + ("idefics2_perceiver_model", "idefics2"), ] ) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index b2fbf35da241..1ff6b96306fd 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -126,6 +126,8 @@ ("ibert", "IBertModel"), ("idefics", "IdeficsModel"), ("idefics2", "Idefics2Model"), + ("idefics2_perceiver_model", "Idefics2PerceiverResampler"), + ("idefics2_vision_model", "Idefics2VisionTransformer"), ("imagegpt", "ImageGPTModel"), ("informer", "InformerModel"), ("jamba", "JambaModel"), diff --git a/src/transformers/models/idefics2/__init__.py b/src/transformers/models/idefics2/__init__.py index 1d8d3e4b571d..13f350150196 100644 --- a/src/transformers/models/idefics2/__init__.py +++ b/src/transformers/models/idefics2/__init__.py @@ -16,7 +16,7 @@ from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available -_import_structure = {"configuration_idefics2": ["Idefics2Config"]} +_import_structure = {"configuration_idefics2": ["Idefics2Config", "Idefics2VisionConfig", "Idefics2PerceiverConfig"]} try: @@ -38,11 +38,13 @@ "Idefics2ForConditionalGeneration", "Idefics2PreTrainedModel", "Idefics2Model", + "Idefics2VisionTransformer", + "Idefics2PerceiverResampler", ] _import_structure["processing_idefics2"] = ["Idefics2Processor"] if TYPE_CHECKING: - from .configuration_idefics2 import Idefics2Config + from .configuration_idefics2 import Idefics2Config, Idefics2PerceiverConfig, Idefics2VisionConfig try: if not is_vision_available(): @@ -61,7 +63,9 @@ from .modeling_idefics2 import ( Idefics2ForConditionalGeneration, Idefics2Model, + Idefics2PerceiverResampler, Idefics2PreTrainedModel, + Idefics2VisionTransformer, ) from .processing_idefics2 import Idefics2Processor diff --git a/src/transformers/models/idefics2/configuration_idefics2.py b/src/transformers/models/idefics2/configuration_idefics2.py index 1333895407e6..360765504f22 100644 --- a/src/transformers/models/idefics2/configuration_idefics2.py +++ b/src/transformers/models/idefics2/configuration_idefics2.py @@ -57,7 +57,7 @@ class Idefics2VisionConfig(PretrainedConfig): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. - intializer_range (`float`, *optional*, defaults to 0.02): + initializer_range (`flloat`, *optional*, defaults to 0.02): The standard deviation for initializing all weight matrices in the model. Example: diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 4284a6114e81..c21993f90bb2 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -571,9 +571,86 @@ def forward( ) -class Idefics2VisionTransformer(nn.Module): +IDEFICS2_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`Idefics2Config`] or [`Idefics2VisionConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare Idefics2 Model outputting raw hidden-states without any specific head on top.", + IDEFICS2_START_DOCSTRING, +) +class Idefics2PreTrainedModel(PreTrainedModel): + config_class = Idefics2Config + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["Idefics2VisionAttention", "Idefics2MLP", "Idefics2PerceiverLayer", "Idefics2DecoderLayer"] + _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_cache_class = True + _is_composite = True + + def _init_weights(self, module): + std = ( + self.config.text_config.initializer_range + if hasattr(self.config, "initializer_range") + else self.config.text_config.initializer_range + ) + + if hasattr(module, "class_embedding"): + module.class_embedding.data.normal_(mean=0.0, std=std) + + if isinstance(module, (nn.Linear, nn.Conv2d)): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +IDEFICS2_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)): + The tensors corresponding to the input images. Pixel values can be obtained using + [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses + [`CLIPImageProcessor`] for processing images). + pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): + Mask to avoid performing attention on padding pixel indices. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + """Idefics2 vision encoder model that returnss raw image embeddings.""", + IDEFICS2_START_DOCSTRING, +) +class Idefics2VisionTransformer(Idefics2PreTrainedModel): + _supports_sdpa = False + _is_composite = False + def __init__(self, config: Idefics2VisionConfig): - super().__init__() + super().__init__(config) embed_dim = config.hidden_size self.config = config @@ -985,15 +1062,32 @@ def forward( return outputs -class Idefics2PerceiverResampler(nn.Module): +IDEFICS2_INPUTS_DOCSTRING = r""" + Args: + context (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`): + The hidden states of the image after vision encoder and modality projection. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) +""" + + +@add_start_docstrings( + "Idefics2 perceiver resampler model that performs `depth` blocks of cross-attention with a fixed ", + "`n_latents` inputs to decrease embedding sequence length. The Resampler acts as a form of learned pooling and ", + "is derived from [Perceiver: General Perception with Iterative Attention](https://arxiv.org/abs/2103.03206)", + IDEFICS2_START_DOCSTRING, +) +class Idefics2PerceiverResampler(Idefics2PreTrainedModel): + _supports_sdpa = False + _is_composite = False + def __init__(self, config) -> None: - """ - Instantiates a Perceiver Resampler that operates over a sequence of embeddings (say from a ResNet or ViT or - MAE) of a given dimension, performs `depth` blocks of cross-attention with a fixed `n_latents` inputs, then - returns a Tensor of shape [bsz, n_latents, embed_dim]. The Resampler acts as a form of learned pooling and - is derived from [Perceiver: General Perception with Iterative Attention](https://arxiv.org/abs/2103.03206). - """ - super().__init__() + super().__init__(config) self.hidden_size = config.text_config.hidden_size self.hidden_act = config.perceiver_config.hidden_act self.n_latents = config.perceiver_config.resampler_n_latents @@ -1007,12 +1101,12 @@ def __init__(self, config) -> None: self.layers = nn.ModuleList([Idefics2PerceiverLayer(config, idx) for idx in range(self.depth)]) self.norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) - self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" + self._use_flash_attention_2 = config.perceiver_config._attn_implementation == "flash_attention_2" def forward( self, context: torch.Tensor, - attention_mask, + attention_mask: torch.Tensor, ) -> torch.Tensor: # seq embed -> bsz seq embed latents = self.latents.unsqueeze(0).expand((context.shape[0], *self.latents.size())) @@ -1063,57 +1157,6 @@ def forward(self, image_hidden_states, attention_mask): return image_hidden_states -IDEFICS2_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`Idefics2Config`] or [`Idefics2VisionConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare Idefics2 Model outputting raw hidden-states without any specific head on top.", - IDEFICS2_START_DOCSTRING, -) -class Idefics2PreTrainedModel(PreTrainedModel): - config_class = Idefics2Config - base_model_prefix = "model" - supports_gradient_checkpointing = True - _no_split_modules = ["Idefics2VisionAttention", "Idefics2MLP", "Idefics2PerceiverLayer", "Idefics2DecoderLayer"] - _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True - _supports_cache_class = True - _is_composite = True - - def _init_weights(self, module): - std = ( - self.config.text_config.initializer_range - if hasattr(self.config, "initializer_range") - else self.config.text_config.initializer_range - ) - - if hasattr(module, "class_embedding"): - module.class_embedding.data.normal_(mean=0.0, std=std) - - if isinstance(module, (nn.Linear, nn.Conv2d)): - module.weight.data.normal_(mean=0.0, std=std) - if module.bias is not None: - module.bias.data.zero_() - elif isinstance(module, nn.Embedding): - module.weight.data.normal_(mean=0.0, std=std) - if module.padding_idx is not None: - module.weight.data[module.padding_idx].zero_() - - IDEFICS2_INPUTS_DOCSTRING = r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index de739c6e7004..24ce57578917 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -4775,6 +4775,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class Idefics2PerceiverResampler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class Idefics2PreTrainedModel(metaclass=DummyObject): _backends = ["torch"] @@ -4789,6 +4796,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class Idefics2VisionTransformer(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class ImageGPTForCausalImageModeling(metaclass=DummyObject): _backends = ["torch"] diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index b85e6d351bea..817c9c5dd0dd 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -350,28 +350,18 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 - # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not - # In this case we know that Idefics2 has 'eager' for all perceiver/vision models and that - # they are not PretrainedModel so we cannot check `_supports_sdpa` cls attr but only set it here - perceiver_attn = "eager" - vision_attn = "eager" + perceiver_attn = "sdpa" if model.connector.perceiver_resampler._supports_sdpa else "eager" + vision_attn = "sdpa" if model.vision_model._supports_sdpa else "eager" self.assertTrue(model_sdpa.config.text_config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) # Also test that nothing break if we request SDPA explicitly - # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible - # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA - # Checking error is out-of-scope of this test - model_sdpa_explicit = model_class.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) - model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - - self.assertTrue(model_sdpa_explicit.config.text_config._attn_implementation == "sdpa") - self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) - self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + # If the model supports sdpa (i.e. one of sub-models supports it) we'll raise error because we + # explicitly asked for SDPA, in comparison to above when SDPA dispatches by default is it is available. + with self.assertRaises(ValueError): + _ = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa") model_eager = model_class.from_pretrained( tmpdirname, @@ -389,7 +379,6 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: raise ValueError("The eager model should not have SDPA attention layers") - print(model_sdpa) has_sdpa = False for name, submodule in model_sdpa.named_modules(): class_name = submodule.__class__.__name__ diff --git a/utils/check_table.py b/utils/check_table.py index f62d6ce08a24..d3aeef78e1b1 100644 --- a/utils/check_table.py +++ b/utils/check_table.py @@ -176,6 +176,8 @@ def _center_text(text: str, width: int) -> str: MODEL_NAMES_TO_IGNORE = [ "CLIPVisionModel", "CLIPTextModel", + "Idefics2VisionTransformer", + "Idefics2PerceiverResampler", "SiglipVisionModel", "SiglipTextModel", "ChineseCLIPVisionModel", From 2713616038ff2b89f111d32a033abc43a2366351 Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 7 Aug 2024 08:43:03 +0200 Subject: [PATCH 33/68] nits --- .../models/idefics/modeling_idefics.py | 1 - tests/models/idefics/test_modeling_idefics.py | 19 +++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/transformers/models/idefics/modeling_idefics.py b/src/transformers/models/idefics/modeling_idefics.py index 9204a039db66..22c9d09b80f3 100644 --- a/src/transformers/models/idefics/modeling_idefics.py +++ b/src/transformers/models/idefics/modeling_idefics.py @@ -941,7 +941,6 @@ class IdeficsPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["IdeficsDecoderLayer", "IdeficsGatedCrossAttentionLayer"] _supports_sdpa = True - _is_composite = True def _init_weights(self, module): # important: this ported version of Idefics isn't meant for training from scratch - only diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index 2ddc7612208f..d3456bdac083 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -594,27 +594,15 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - # see https://github.com/huggingface/transformers/pull/32238 - # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not - # In this case we get SDPA by default if it `_supports_sdpa` else fallback to "eager" - # and we know that Idefics's perceiver and vision models are simple nn.Modules wo SDPA - perceiver_attn = "eager" - vision_attn = "eager" - - self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) - self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") # Also test that nothing break if we request SDPA explicitly - # Of the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible - # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA - # Checking error is out-of-scope of this test model_sdpa_explicit = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" ) model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue(model_sdpa_explicit.config.perceiver_config._attn_implementation == perceiver_attn) - self.assertTrue(model_sdpa_explicit.config.vision_config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") model_eager = model_class.from_pretrained( tmpdirname, @@ -623,8 +611,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config.perceiver_config._attn_implementation == "eager") - self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ From 3aef7632a3c7e8c81109abe4ac54ccd77882b581 Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 7 Aug 2024 08:52:34 +0200 Subject: [PATCH 34/68] skip tests --- utils/check_repo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/check_repo.py b/utils/check_repo.py index 26e40112bf89..938cd3e761ff 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -128,6 +128,8 @@ "SeamlessM4TCodeHifiGan", # Building part of bigger (tested) model. "SeamlessM4TTextToUnitForConditionalGeneration", # Building part of bigger (tested) model. "ChameleonVQVAE", # VQVAE here is used only for encoding (discretizing) and is tested as part of bigger model + "Idefics2VisionTransformer", # Idefics2 modules are tested as part of a bigger model + "Idefics2PerceiverResampler", # Idefics2 modules are tested as part of a bigger model ] # Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't From 6c31934986318e13e96af9542040b5bbf6a3ed3b Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 7 Aug 2024 09:42:32 +0200 Subject: [PATCH 35/68] doctests --- docs/source/en/model_doc/idefics2.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/source/en/model_doc/idefics2.md b/docs/source/en/model_doc/idefics2.md index 5ad56b7b5c52..3ba5bae786ce 100644 --- a/docs/source/en/model_doc/idefics2.md +++ b/docs/source/en/model_doc/idefics2.md @@ -195,12 +195,29 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h [[autodoc]] Idefics2Config +## Idefics2VisionConfig + +[[autodoc]] Idefics2VisionConfig + +## Idefics2PerceiverConfig + +[[autodoc]] Idefics2PerceiverConfig + ## Idefics2Model [[autodoc]] Idefics2Model - forward +## Idefics2VisionTransformer + +[[autodoc]] Idefics2VisionTransformer + - forward + +## Idefics2PerceiverResampler + +[[autodoc]] Idefics2PerceiverResampler + - forward ## Idefics2ForConditionalGeneration From 2f9017623fef2db868edcce5545ff4d9d3688794 Mon Sep 17 00:00:00 2001 From: Raushan Turganbay Date: Thu, 8 Aug 2024 12:42:04 +0500 Subject: [PATCH 36/68] Update src/transformers/models/idefics2/configuration_idefics2.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- src/transformers/models/idefics2/configuration_idefics2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/idefics2/configuration_idefics2.py b/src/transformers/models/idefics2/configuration_idefics2.py index 360765504f22..69bdcf6d2bce 100644 --- a/src/transformers/models/idefics2/configuration_idefics2.py +++ b/src/transformers/models/idefics2/configuration_idefics2.py @@ -57,7 +57,7 @@ class Idefics2VisionConfig(PretrainedConfig): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. - initializer_range (`flloat`, *optional*, defaults to 0.02): + initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation for initializing all weight matrices in the model. Example: From 3dfb48ce16b10ffb42eefb52d381f81f6a29807a Mon Sep 17 00:00:00 2001 From: Raushan Turganbay Date: Thu, 8 Aug 2024 12:45:40 +0500 Subject: [PATCH 37/68] Update tests/models/clip/test_modeling_clip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- tests/models/clip/test_modeling_clip.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/models/clip/test_modeling_clip.py b/tests/models/clip/test_modeling_clip.py index ba85c7bc4468..e6b0af5ef0dc 100644 --- a/tests/models/clip/test_modeling_clip.py +++ b/tests/models/clip/test_modeling_clip.py @@ -257,9 +257,8 @@ def get_mean_reldiff(msg, current_case, x, ref, atol, rtol): self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_eager.config._attn_implementation == "eager") else: - vision_attn = text_attn = ( - "sdpa" if model._supports_sdpa else "eager" - ) # sigLip has one shared cls attr for all models + # sigLip has one shared cls attr for all models + vision_attn = text_attn = "sdpa" if model._supports_sdpa else "eager" self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) self.assertTrue(model_sdpa.config.text_config._attn_implementation == text_attn) self.assertTrue( From 505ed3fd0a38a0fa981904d8b009b4127f8403c6 Mon Sep 17 00:00:00 2001 From: Raushan Turganbay Date: Thu, 8 Aug 2024 14:54:00 +0500 Subject: [PATCH 38/68] Update tests/models/idefics2/test_modeling_idefics2.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- tests/models/idefics2/test_modeling_idefics2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 817c9c5dd0dd..2b17bae6d8ce 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -403,7 +403,7 @@ class Idefics2ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTest test_torchscript = False is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because LM/vision models used in tests dont support SDPA supports_sdpa = False From 4c9f8945b47c0fc70576ff4261d1bec8bdd35033 Mon Sep 17 00:00:00 2001 From: Raushan Turganbay Date: Thu, 8 Aug 2024 14:54:08 +0500 Subject: [PATCH 39/68] Update tests/models/idefics2/test_modeling_idefics2.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- tests/models/idefics2/test_modeling_idefics2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 2b17bae6d8ce..55b024311dc4 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -405,7 +405,7 @@ class Idefics2ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTest is_multimodal = True # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because LM/vision models used in tests dont support SDPA + # This flag is used by tests and is set to False because LM/vision models used in tests don't support SDPA supports_sdpa = False def setUp(self): From d7d54f8e5eb255083ff384bbda8cab6f6eabab85 Mon Sep 17 00:00:00 2001 From: Raushan Turganbay Date: Thu, 8 Aug 2024 14:57:48 +0500 Subject: [PATCH 40/68] Update src/transformers/configuration_utils.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- src/transformers/configuration_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index 0a5440c14022..a41f176faaa2 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -832,8 +832,8 @@ def __repr__(self): return f"{self.__class__.__name__} {self.to_json_string()}" def __iter__(self): - for attr, value in copy.deepcopy(self.__dict__).items(): - yield attr, value + for attr in copy.deepcopy(self.__dict__): + yield attr def to_diff_dict(self) -> Dict[str, Any]: """ From b6e9951eac1cc908dfa50cb039eddbe07dabdb0f Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 8 Aug 2024 15:29:18 +0200 Subject: [PATCH 41/68] major update, no need for automap --- src/transformers/modeling_utils.py | 34 ++---- .../models/blip_2/modeling_blip_2.py | 11 +- src/transformers/models/clip/modeling_clip.py | 11 +- .../modeling_encoder_decoder.py | 6 +- .../models/idefics2/modeling_idefics2.py | 18 ++-- .../instructblip/modeling_instructblip.py | 10 +- .../modeling_instructblipvideo.py | 11 +- .../models/llava/modeling_llava.py | 4 +- .../models/llava_next/modeling_llava_next.py | 4 +- .../modeling_llava_next_video.py | 4 +- .../models/musicgen/modeling_musicgen.py | 15 ++- .../modeling_musicgen_melody.py | 15 ++- .../models/paligemma/modeling_paligemma.py | 4 +- .../models/siglip/modeling_siglip.py | 9 +- .../modeling_speech_encoder_decoder.py | 7 +- .../video_llava/modeling_video_llava.py | 6 +- .../models/vipllava/modeling_vipllava.py | 4 +- .../modeling_vision_encoder_decoder.py | 6 +- .../modeling_vision_text_dual_encoder.py | 6 +- tests/models/blip_2/test_modeling_blip_2.py | 2 - tests/models/clip/test_modeling_clip.py | 17 +-- .../test_modeling_encoder_decoder.py | 28 ++--- .../models/idefics2/test_modeling_idefics2.py | 24 +++-- .../models/musicgen/test_modeling_musicgen.py | 31 +++--- .../test_modeling_musicgen_melody.py | 27 +++-- tests/models/siglip/test_modeling_siglip.py | 22 ++-- .../test_modeling_speech_encoder_decoder.py | 102 +++++++++++++++++- .../test_modeling_vision_encoder_decoder.py | 38 +++---- tests/test_modeling_common.py | 44 ++++---- 29 files changed, 325 insertions(+), 195 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index bd28f513d701..0aeb91b61001 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -45,7 +45,6 @@ from .dynamic_module_utils import custom_object_save from .generation import GenerationConfig, GenerationMixin from .integrations import PeftAdapterMixin, deepspeed_config, is_deepspeed_zero3_enabled -from .models.auto.modeling_auto import MODEL_MAPPING from .pytorch_utils import ( # noqa: F401 Conv1D, apply_chunking_to_forward, @@ -1524,32 +1523,21 @@ def _autoset_attn_implementation( # we have to check and dispatch SDPA to each sub-config, in case any of them support it. # If one sub-model supports SDPA while other doesn't, an error will be raised following the # typical SDPA-dispatch path (i.e. if hard_check). Same goes for FA2. - sub_configs = {key: value for key, value in config if isinstance(value, PretrainedConfig)} + sub_configs = { + key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) + } if sub_configs: attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): - sub_model_cls = MODEL_MAPPING.get(type(sub_config), None) - if sub_model_cls is not None: - # User can pass attn_implementation={"vision_config": "sdpa", "text_config": "eager"} - # as well as attn_implementation="sdpa", which means sdpa in all sub-configs - if isinstance(requested_attn_implementation, dict) and key in requested_attn_implementation: - sub_config._attn_implementation = requested_attn_implementation[key] - else: - sub_config._attn_implementation = requested_attn_implementation - sub_model_cls._autoset_attn_implementation( - sub_config, - use_flash_attention_2=use_flash_attention_2, - torch_dtype=torch_dtype, - device_map=device_map, - check_device_map=check_device_map, - ) - setattr(config, key, sub_config) - attn_implementation_per_subconfig[key] = sub_config._attn_implementation + attn_implementation_per_subconfig[key] = ( + requested_attn_implementation + if not isinstance(requested_attn_implementation, dict) + else requested_attn_implementation.get(key) + ) - # Set the general attn_implementation to a dict where keys are sub-configs - requested_attn_implementation = ( - attn_implementation_per_subconfig if cls._is_composite else requested_attn_implementation - ) + if cls._is_composite: + config._attn_implementation = attn_implementation_per_subconfig + requested_attn_implementation = config._attn_implementation if use_flash_attention_2: logger.warning_once( diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index e335013e52df..aa60ce9be272 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -539,6 +539,9 @@ class Blip2VisionModel(Blip2PreTrainedModel): main_input_name = "pixel_values" config_class = Blip2VisionConfig + # Ignore copy + _is_composite = False + def __init__(self, config: Blip2VisionConfig): super().__init__(config) self.config = config @@ -1029,6 +1032,8 @@ class Blip2QFormerModel(Blip2PreTrainedModel): Querying Transformer (Q-Former), used in BLIP-2. """ + _is_composite = False + def __init__(self, config: Blip2QFormerConfig): super().__init__(config) self.config = config @@ -1228,7 +1233,7 @@ def __init__(self, config: Blip2Config): super().__init__(config) self.vision_model = Blip2VisionModel._from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) @@ -1237,11 +1242,11 @@ def __init__(self, config: Blip2Config): self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) # Update _tied_weights_keys using the base model used. diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index cf784a2d7686..8fdb183acbcb 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -939,6 +939,7 @@ def forward( ) class CLIPTextModel(CLIPPreTrainedModel): config_class = CLIPTextConfig + _is_composite = False _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer"] @@ -1060,6 +1061,7 @@ class CLIPVisionModel(CLIPPreTrainedModel): config_class = CLIPVisionConfig main_input_name = "pixel_values" _no_split_modules = ["CLIPEncoderLayer"] + _is_composite = False def __init__(self, config: CLIPVisionConfig): super().__init__(config) @@ -1139,12 +1141,12 @@ def __init__(self, config: CLIPConfig): self.vision_embed_dim = vision_config.hidden_size text_model = CLIPTextModel._from_config( - text_config, attn_implementation=config.text_config._attn_implementation + text_config, attn_implementation=config._attn_implementation["text_config"] ) self.text_model = text_model.text_model vision_model = CLIPVisionModel._from_config( - vision_config, attn_implementation=config.vision_config._attn_implementation + vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.vision_model = vision_model.vision_model @@ -1355,6 +1357,7 @@ def forward( ) class CLIPTextModelWithProjection(CLIPPreTrainedModel): config_class = CLIPTextConfig + _is_composite = False _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer"] @@ -1438,6 +1441,7 @@ def forward( class CLIPVisionModelWithProjection(CLIPPreTrainedModel): config_class = CLIPVisionConfig main_input_name = "pixel_values" + _is_composite = False def __init__(self, config: CLIPVisionConfig): super().__init__(config) @@ -1517,13 +1521,14 @@ def forward( ) class CLIPForImageClassification(CLIPPreTrainedModel): main_input_name = "pixel_values" + _is_composite = False def __init__(self, config: CLIPConfig) -> None: super().__init__(config) self.num_labels = config.num_labels vision_model = CLIPVisionModel._from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation ) self.vision_model = vision_model.vision_model diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index 2e3681318d24..93488c27393d 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -210,13 +210,13 @@ def __init__( if encoder is None: from ..auto.modeling_auto import AutoModel - encoder = AutoModel.from_config(config.encoder, attn_implementation=config.encoder._attn_implementation) + encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation["encoder"]) if decoder is None: from ..auto.modeling_auto import AutoModelForCausalLM decoder = AutoModelForCausalLM.from_config( - config.decoder, attn_implementation=config.decoder._attn_implementation + config.decoder, attn_implementation=config._attn_implementation["decoder"] ) self.encoder = encoder @@ -235,6 +235,8 @@ def __init__( # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced + self.config.encoder._attn_implementation = self.encoder.config._attn_implementation + self.config.decoder._attn_implementation = self.decoder.config._attn_implementation self.encoder.config = self.config.encoder self.decoder.config = self.config.decoder diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index c21993f90bb2..b0814f676b35 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -998,9 +998,7 @@ def __init__(self, config, layer_idx: int): self.input_latents_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.input_context_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) - self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[config.perceiver_config._attn_implementation]( - config, layer_idx=layer_idx - ) + self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx) self.post_attention_layernorm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.mlp = Idefics2MLP( hidden_size=config.text_config.hidden_size, @@ -1101,7 +1099,7 @@ def __init__(self, config) -> None: self.layers = nn.ModuleList([Idefics2PerceiverLayer(config, idx) for idx in range(self.depth)]) self.norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) - self._use_flash_attention_2 = config.perceiver_config._attn_implementation == "flash_attention_2" + self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" def forward( self, @@ -1149,7 +1147,9 @@ def __init__(self, config): output_size=config.text_config.hidden_size, hidden_act=config.text_config.hidden_act, ) - self.perceiver_resampler = Idefics2PerceiverResampler(config) + self.perceiver_resampler = Idefics2PerceiverResampler._from_config( + config, attn_implementation=config._attn_implementation["perceiver_config"] + ) def forward(self, image_hidden_states, attention_mask): image_hidden_states = self.modality_projection(image_hidden_states) @@ -1237,16 +1237,18 @@ def __init__(self, config: Idefics2Config): self.padding_idx = self.config.text_config.pad_token_id self.vocab_size = self.config.text_config.vocab_size - self.vision_model = Idefics2VisionTransformer(config.vision_config) + self.vision_model = Idefics2VisionTransformer._from_config( + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] + ) self.connector = Idefics2Connector(config) self.text_model = AutoModel.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.image_seq_len = config.perceiver_config.resampler_n_latents self.image_token_id = self.config.image_token_id - self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" + self._use_flash_attention_2 = config._attn_implementation["text_config"] == "flash_attention_2" self.post_init() diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 4ba0fc1bf1b8..97c4474437cb 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -527,6 +527,8 @@ def forward( class InstructBlipVisionModel(InstructBlipPreTrainedModel): main_input_name = "pixel_values" config_class = InstructBlipVisionConfig + # Ignore copy + _is_composite = False def __init__(self, config: InstructBlipVisionConfig): super().__init__(config) @@ -1076,6 +1078,8 @@ class InstructBlipQFormerModel(InstructBlipPreTrainedModel): instruction as input. """ + _is_composite = False + def __init__(self, config: InstructBlipQFormerConfig): super().__init__(config) self.config = config @@ -1284,7 +1288,7 @@ def __init__(self, config: InstructBlipConfig): super().__init__(config) self.vision_model = InstructBlipVisionModel._from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) @@ -1294,11 +1298,11 @@ def __init__(self, config: InstructBlipConfig): if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) if language_model._no_split_modules is not None: diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 2254eff9d21c..7e842a670b16 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -537,6 +537,9 @@ class InstructBlipVideoVisionModel(InstructBlipVideoPreTrainedModel): main_input_name = "pixel_values" config_class = InstructBlipVideoVisionConfig + # Ignore copy + _is_composite = False + def __init__(self, config: InstructBlipVideoVisionConfig): super().__init__(config) self.config = config @@ -1085,6 +1088,8 @@ class InstructBlipVideoQFormerModel(InstructBlipVideoPreTrainedModel): instruction as input. """ + _is_composite = True + def __init__(self, config: InstructBlipVideoQFormerConfig): super().__init__(config) self.config = config @@ -1293,7 +1298,7 @@ def __init__(self, config: InstructBlipVideoConfig): super().__init__(config) self.vision_model = InstructBlipVideoVisionModel._from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) @@ -1303,11 +1308,11 @@ def __init__(self, config: InstructBlipVideoConfig): if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) if language_model._no_split_modules is not None: diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 57de61b731c7..e81b4d2f2194 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -230,13 +230,13 @@ class LlavaForConditionalGeneration(LlavaPreTrainedModel): def __init__(self, config: LlavaConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.multi_modal_projector = LlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index b7ad164d123d..86f5d2578787 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -339,7 +339,7 @@ class LlavaNextForConditionalGeneration(LlavaNextPreTrainedModel): def __init__(self, config: LlavaNextConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.multi_modal_projector = LlavaNextMultiModalProjector(config) @@ -348,7 +348,7 @@ def __init__(self, config: LlavaNextConfig): self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index b275e1fd200e..8cc80035c215 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -382,7 +382,7 @@ def __init__( ): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.multi_modal_projector = LlavaNextVideoMultiModalProjector(config) @@ -391,7 +391,7 @@ def __init__( self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/src/transformers/models/musicgen/modeling_musicgen.py b/src/transformers/models/musicgen/modeling_musicgen.py index 131195c847bb..cb9086b93892 100644 --- a/src/transformers/models/musicgen/modeling_musicgen.py +++ b/src/transformers/models/musicgen/modeling_musicgen.py @@ -1708,15 +1708,21 @@ def __init__( if text_encoder is None: from ..auto.modeling_auto import AutoModelForTextEncoding - text_encoder = AutoModelForTextEncoding.from_config(config.text_encoder) + text_encoder = AutoModelForTextEncoding.from_config( + config.text_encoder, attn_implementation=config._attn_implementation["text_encoder"] + ) if audio_encoder is None: from ..auto.modeling_auto import AutoModel - audio_encoder = AutoModel.from_config(config.audio_encoder) + audio_encoder = AutoModel.from_config( + config.audio_encoder, attn_implementation=config._attn_implementation["audio_encoder"] + ) if decoder is None: - decoder = MusicgenForCausalLM(config.decoder) + decoder = MusicgenForCausalLM._from_config( + config.decoder, attn_implementation=config._attn_implementation["decoder"] + ) self.text_encoder = text_encoder self.audio_encoder = audio_encoder @@ -1740,6 +1746,9 @@ def __init__( # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced + self.config.text_encoder._attn_implementation = self.text_encoder.config._attn_implementation + self.config.audio_encoder._attn_implementation = self.audio_encoder.config._attn_implementation + self.config.decoder._attn_implementation = self.decoder.config._attn_implementation self.text_encoder.config = self.config.text_encoder self.audio_encoder.config = self.config.audio_encoder self.decoder.config = self.config.decoder diff --git a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py index 57fa643388d3..a1ce7b3f24b2 100644 --- a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py +++ b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py @@ -1625,13 +1625,19 @@ def __init__( super().__init__(config) if text_encoder is None: - text_encoder = AutoModelForTextEncoding.from_config(config.text_encoder) + text_encoder = AutoModelForTextEncoding.from_config( + config.text_encoder, attn_implementation=config._attn_implementation["text_encoder"] + ) if audio_encoder is None: - audio_encoder = AutoModel.from_config(config.audio_encoder) + audio_encoder = AutoModel.from_config( + config.audio_encoder, attn_implementation=config._attn_implementation["audio_encoder"] + ) if decoder is None: - decoder = MusicgenMelodyForCausalLM(config.decoder) + decoder = MusicgenMelodyForCausalLM._from_config( + config.decoder, attn_implementation=config._attn_implementation["decoder"] + ) self.text_encoder = text_encoder self.audio_encoder = audio_encoder @@ -1639,6 +1645,9 @@ def __init__( # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced + self.config.text_encoder._attn_implementation = self.text_encoder.config._attn_implementation + self.config.audio_encoder._attn_implementation = self.audio_encoder.config._attn_implementation + self.config.decoder._attn_implementation = self.decoder.config._attn_implementation self.text_encoder.config = self.config.text_encoder self.audio_encoder.config = self.config.audio_encoder self.decoder.config = self.config.decoder diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 57fa9c1e3b71..ea17fe591854 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -224,14 +224,14 @@ class PaliGemmaForConditionalGeneration(PaliGemmaPreTrainedModel): def __init__(self, config: PaliGemmaConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config=config.vision_config, attn_implementation=config.vision_config._attn_implementation + config=config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.multi_modal_projector = PaliGemmaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self._attn_implementation = config._attn_implementation language_model = AutoModelForCausalLM.from_config( - config=config.text_config, attn_implementation=config.text_config._attn_implementation + config=config.text_config, attn_implementation=config._attn_implementation["text_config"] ) if language_model._tied_weights_keys is not None: diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 9cb94935ba7d..a27da11b690b 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -999,6 +999,7 @@ def forward( ) class SiglipTextModel(SiglipPreTrainedModel): config_class = SiglipTextConfig + _is_composite = False def __init__(self, config: SiglipTextConfig): super().__init__(config) @@ -1141,6 +1142,7 @@ def forward(self, hidden_state): class SiglipVisionModel(SiglipPreTrainedModel): config_class = SiglipVisionConfig main_input_name = "pixel_values" + _is_composite = False def __init__(self, config: SiglipVisionConfig): super().__init__(config) @@ -1220,10 +1222,10 @@ def __init__(self, config: SiglipConfig): # First, initialize the text and vision models with proper attention implementation text_model = SiglipTextModel._from_config( - text_config, attn_implementation=config.text_config._attn_implementation + text_config, attn_implementation=config._attn_implementation["text_config"] ) vision_model = SiglipVisionModel._from_config( - vision_config, attn_implementation=config.vision_config._attn_implementation + vision_config, attn_implementation=config._attn_implementation["vision_config"] ) # Second, get the text and vision submodules (for backward compatibility) @@ -1451,6 +1453,7 @@ def forward( ) class SiglipForImageClassification(SiglipPreTrainedModel): main_input_name = "pixel_values" + _is_composite = False def __init__(self, config: SiglipConfig) -> None: super().__init__(config) @@ -1460,7 +1463,7 @@ def __init__(self, config: SiglipConfig) -> None: # Create the vision model with proper attention # and take only vision_model submodule (for backward compatibility) vision_model = SiglipVisionModel._from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation ) self.vision_model = vision_model.vision_model diff --git a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py index 3eece905f034..8d1b86908b75 100644 --- a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py +++ b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py @@ -182,6 +182,7 @@ class SpeechEncoderDecoderModel(PreTrainedModel): main_input_name = "inputs" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False + _is_composite = True def __init__( self, @@ -212,11 +213,11 @@ def __init__( super().__init__(config) if encoder is None: - encoder = AutoModel.from_config(config.encoder, attn_implementation=config.encoder._attn_implementation) + encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation["encoder"]) if decoder is None: decoder = AutoModelForCausalLM.from_config( - config.decoder, attn_implementation=config.decoder._attn_implementation + config.decoder, attn_implementation=config._attn_implementation["decoder"] ) self.encoder = encoder @@ -235,6 +236,8 @@ def __init__( # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced + self.config.encoder._attn_implementation = self.encoder.config._attn_implementation + self.config.decoder._attn_implementation = self.decoder.config._attn_implementation self.encoder.config = self.config.encoder self.decoder.config = self.config.decoder diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 8df37577fe75..7a46fed1dcc9 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -239,16 +239,16 @@ class VideoLlavaForConditionalGeneration(VideoLlavaPreTrainedModel): def __init__(self, config: VideoLlavaConfig): super().__init__(config) self.video_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.image_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.multi_modal_projector = VideoLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 88a3690a3a28..0de1825071c5 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -235,13 +235,13 @@ class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel): def __init__(self, config: VipLlavaConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.multi_modal_projector = VipLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py index 1910c9a8912d..353c332d3ee9 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py @@ -191,11 +191,11 @@ def __init__( super().__init__(config) if encoder is None: - encoder = AutoModel.from_config(config.encoder, attn_implementation=config.encoder._attn_implementation) + encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation["encoder"]) if decoder is None: decoder = AutoModelForCausalLM.from_config( - config.decoder, attn_implementation=config.decoder._attn_implementation + config.decoder, attn_implementation=config._attn_implementation["decoder"] ) self.encoder = encoder @@ -214,6 +214,8 @@ def __init__( # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced + self.config.encoder._attn_implementation = self.encoder.config._attn_implementation + self.config.decoder._attn_implementation = self.decoder.config._attn_implementation self.encoder.config = self.config.encoder self.decoder.config = self.config.decoder diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index f54e94b53358..4a4e68d58579 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -186,12 +186,12 @@ def __init__( vision_model = CLIPVisionModel(config.vision_config) else: vision_model = AutoModel.from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) if text_model is None: text_model = AutoModel.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.vision_model = vision_model @@ -199,6 +199,8 @@ def __init__( # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced + self.config.vision_config._attn_implementation = self.vision_config.config._attn_implementation + self.config.text_config._attn_implementation = self.text_config.config._attn_implementation self.vision_model.config = self.config.vision_config self.text_model.config = self.config.text_config diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index b13110268cc9..b7b091991d7a 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -445,7 +445,6 @@ class Blip2ForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, GenerationT test_attention_outputs = False test_torchscript = False - is_multimodal = True # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA @@ -710,7 +709,6 @@ class Blip2ModelTest(ModelTesterMixin, PipelineTesterMixin, GenerationTesterMixi test_attention_outputs = False test_torchscript = False - is_multimodal = True # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA diff --git a/tests/models/clip/test_modeling_clip.py b/tests/models/clip/test_modeling_clip.py index e6b0af5ef0dc..fe1633c2af7c 100644 --- a/tests/models/clip/test_modeling_clip.py +++ b/tests/models/clip/test_modeling_clip.py @@ -257,16 +257,16 @@ def get_mean_reldiff(msg, current_case, x, ref, atol, rtol): self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_eager.config._attn_implementation == "eager") else: - # sigLip has one shared cls attr for all models + # CLIP has one shared cls attr for all models, so both submodels are SDPA or eager + # We expect `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) vision_attn = text_attn = "sdpa" if model._supports_sdpa else "eager" - self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) - self.assertTrue(model_sdpa.config.text_config._attn_implementation == text_attn) - self.assertTrue( - model_sdpa.config._attn_implementation == {"text_config": text_attn, "vision_config": vision_attn} - ) + self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.text_model.config._attn_implementation == text_attn) + self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) - self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") - self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") self.assertTrue( model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} ) @@ -1156,6 +1156,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("logits",), use_attention_mask_options=(None,), + is_composite=False, ) diff --git a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py index e973bc47f09c..e8b47f35a67d 100644 --- a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py +++ b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py @@ -718,20 +718,17 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 - # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not - # In this case we get SDPA by default if it `_supports_Sdpa` else fallback to "eager" + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - - self.assertTrue( - model_sdpa.config._attn_implementation == {"encoder": encoder_attn, "decoder": decoder_attn} - ) - self.assertTrue(model_sdpa.config.encoder._attn_implementation == encoder_attn) - self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + self.assertTrue(model_sdpa.config._attn_implementation == {"encoder": None, "decoder": None}) + self.assertTrue(model_sdpa.encoder.config._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) # Also test that nothing break if we request SDPA explicitly, when both sub-parts support it. - # If the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible - # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA + # If the model supports sdpa (i.e. all of sub-models supports it) we'll dispatch safely + # Otherwise we should raise error that SDPA is not supported, as some of the sub-models doesn't support if encoder_attn == "sdpa" and decoder_attn == "sdpa": model_sdpa_explicit = EncoderDecoderModel.from_pretrained( tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" @@ -742,8 +739,11 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_sdpa_explicit.config._attn_implementation == {"encoder": encoder_attn, "decoder": decoder_attn} ) - self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) - self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) + else: + with self.assertRaises(ValueError): + model_sdpa_explicit = EncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) model_eager = EncoderDecoderModel.from_pretrained( tmpdirname, @@ -753,8 +753,8 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_eager = model_eager.eval().to(torch_device) self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) - self.assertTrue(model_eager.config.encoder._attn_implementation == "eager") - self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + self.assertTrue(model_eager.encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 55b024311dc4..24c2dcdf808e 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -350,12 +350,15 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 - perceiver_attn = "sdpa" if model.connector.perceiver_resampler._supports_sdpa else "eager" - vision_attn = "sdpa" if model.vision_model._supports_sdpa else "eager" - - self.assertTrue(model_sdpa.config.text_config._attn_implementation == "sdpa") - self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == perceiver_attn) - self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) + # we put `None` because that is the requested attn implementation, which will dispatch to SDPA internally if available + perceiver_attn = None if model.connector.perceiver_resampler._supports_sdpa else "eager" + vision_attn = None if model.vision_model._supports_sdpa else "eager" + self.assertTrue( + model_sdpa.config._attn_implementation + == {"text_config": None, "perceiver_config": None, "vision_config": None} + ) + self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.connector.perceiver_resampler.config._attn_implementation == perceiver_attn) # Also test that nothing break if we request SDPA explicitly # If the model supports sdpa (i.e. one of sub-models supports it) we'll raise error because we @@ -370,9 +373,12 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") - self.assertTrue(model_sdpa.config.perceiver_config._attn_implementation == "eager") - self.assertTrue(model_sdpa.config.vision_config._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation + == {"text_config": "eager", "perceiver_config": "eager", "vision_config": "eager"} + ) + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.connector.perceiver_resampler.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index c559d5317332..def4bb5e1199 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -622,14 +622,11 @@ def test_flash_attn_2_generate_use_cache(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( - self.is_multimodal and not self.supports_sdpa - ): + if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -1939,14 +1936,11 @@ def test_flash_attn_2_generate_use_cache(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( - self.is_multimodal and not self.supports_sdpa - ): + if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -2007,15 +2001,18 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - self.assertTrue(model_sdpa.config.audio_encoder._attn_implementation == audio_encoder_attn) - self.assertTrue(model_sdpa.config.text_encoder._attn_implementation == text_encoder_attn) - self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) + self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) self.assertTrue( model_sdpa.config._attn_implementation == { - "audio_encoder": audio_encoder_attn, - "text_encoder": text_encoder_attn, - "decoder": decoder_attn, + "audio_encoder": None, + "text_encoder": None, + "decoder": None, } ) @@ -2026,9 +2023,9 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config.audio_encoder._attn_implementation == "eager") - self.assertTrue(model_eager.config.text_encoder._attn_implementation == "eager") - self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") self.assertTrue( model_eager.config._attn_implementation == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index b65dec4f43c4..7d84da2fe35b 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -624,14 +624,11 @@ def test_flash_attn_2_generate_use_cache(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow - # Copied from tests.models.musicgen.test_modeling_musicgen.MusicgenDecoderTest.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( - self.is_multimodal and not self.supports_sdpa - ): + if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -1923,7 +1920,6 @@ def test_flash_attn_2_generate_use_cache(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") @@ -1986,15 +1982,18 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - self.assertTrue(model_sdpa.config.audio_encoder._attn_implementation == audio_encoder_attn) - self.assertTrue(model_sdpa.config.text_encoder._attn_implementation == text_encoder_attn) - self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) + self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) self.assertTrue( model_sdpa.config._attn_implementation == { - "audio_encoder": audio_encoder_attn, - "text_encoder": text_encoder_attn, - "decoder": decoder_attn, + "audio_encoder": None, + "text_encoder": None, + "decoder": None, } ) @@ -2005,9 +2004,9 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config.audio_encoder._attn_implementation == "eager") - self.assertTrue(model_eager.config.text_encoder._attn_implementation == "eager") - self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") self.assertTrue( model_eager.config._attn_implementation == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} diff --git a/tests/models/siglip/test_modeling_siglip.py b/tests/models/siglip/test_modeling_siglip.py index 480994b84092..47ba43e49142 100644 --- a/tests/models/siglip/test_modeling_siglip.py +++ b/tests/models/siglip/test_modeling_siglip.py @@ -137,17 +137,17 @@ def get_mean_reldiff(msg, current_case, x, ref, atol, rtol): self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_eager.config._attn_implementation == "eager") else: - vision_attn = text_attn = ( - "sdpa" if model._supports_sdpa else "eager" - ) # sigLip has one shared cls attr for all models - self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) - self.assertTrue(model_sdpa.config.text_config._attn_implementation == text_attn) - self.assertTrue( - model_sdpa.config._attn_implementation == {"text_config": text_attn, "vision_config": vision_attn} - ) + # SigLip has one shared cls attr for all models, so we assign both submodels heer + vision_attn = text_attn = "sdpa" if model._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.text_model.config._attn_implementation == text_attn) + self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) - self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") - self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") self.assertTrue( model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} ) @@ -941,7 +941,7 @@ def test_initialization(self): @is_flaky() def test_eager_matches_sdpa_inference(self, torch_dtype: str): super().test_eager_matches_sdpa_inference( - torch_dtype=torch_dtype, logit_keys=("logits",), use_attention_mask_options=(False,) + torch_dtype=torch_dtype, logit_keys=("logits",), use_attention_mask_options=(False,), is_composite=False ) diff --git a/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py b/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py index b193cacfb400..e81210ab7914 100644 --- a/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py +++ b/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py @@ -17,8 +17,20 @@ import tempfile import unittest +from parameterized import parameterized + from transformers import is_torch_available -from transformers.testing_utils import require_deterministic_for_xpu, require_torch, slow, torch_device +from transformers.testing_utils import ( + require_deterministic_for_xpu, + require_torch, + require_torch_sdpa, + slow, + torch_device, +) +from transformers.utils import ( + is_torch_bf16_available_on_device, + is_torch_fp16_available_on_device, +) from ...test_modeling_common import floats_tensor, ids_tensor, random_attention_mask from ..bert.test_modeling_bert import BertModelTester @@ -441,6 +453,94 @@ def test_real_model_save_load_from_pretrained(self): max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @require_torch_sdpa + @slow + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + # if not self.supports_sdpa: + # self.skipTest("SDPA is not supported") + + if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): + self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") + + if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): + self.skipTest( + f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" + ) + + # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. + if torch_dtype == "float16": + torch_dtype = torch.float16 + elif torch_dtype == "bfloat16": + torch_dtype = torch.bfloat16 + elif torch_dtype == "float32": + torch_dtype = torch.float32 + + inputs_dict = self.prepare_config_and_inputs() + encoder_config, decoder_config = inputs_dict["config"], inputs_dict["decoder_config"] + config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs( + encoder_config=encoder_config, decoder_config=decoder_config + ) + model = SpeechEncoderDecoderModel(config=config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = SpeechEncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = model_sdpa.eval().to(torch_device) + + # see https://github.com/huggingface/transformers/pull/32238 + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + self.assertTrue(model_sdpa.config._attn_implementation == {"encoder": None, "decoder": None}) + self.assertTrue(model_sdpa.encoder.config._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) + + # Also test that nothing break if we request SDPA explicitly, when both sub-parts support it. + # If the model supports sdpa (i.e. all of sub-models supports it) we'll dispatch safely + # Otherwise we should raise error that SDPA is not supported, as some of the sub-models doesn't support + if encoder_attn == "sdpa" and decoder_attn == "sdpa": + model_sdpa_explicit = SpeechEncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) + model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) + + self.assertTrue( + model_sdpa_explicit.config._attn_implementation + == {"encoder": encoder_attn, "decoder": decoder_attn} + ) + else: + with self.assertRaises(ValueError): + model_sdpa_explicit = SpeechEncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) + + model_eager = SpeechEncoderDecoderModel.from_pretrained( + tmpdirname, + torch_dtype=torch_dtype, + attn_implementation="eager", + ) + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) + self.assertTrue(model_eager.encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa: + raise ValueError("The SDPA model should have SDPA attention layers") + @require_torch class Wav2Vec2BertModelTest(EncoderDecoderMixin, unittest.TestCase): diff --git a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py index 69addc0e8ec5..8593f0494191 100644 --- a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py +++ b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py @@ -84,8 +84,7 @@ @require_torch class EncoderDecoderMixin: - has_attentions: bool = False - supports_sdpa: bool = False + supports_sdpa = False def get_encoder_decoder_model(self, config, decoder_config): pass @@ -393,9 +392,6 @@ def test_real_model_save_load_from_pretrained(self): @require_torch_sdpa @slow def test_eager_matches_sdpa_inference(self, torch_dtype: str): - if not self.has_attentions: - self.skipTest(reason="Model architecture does not support attentions") - if not self.supports_sdpa: self.skipTest("SDPA is not supported") @@ -428,20 +424,17 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 - # TL:DR; each sub-config will dispatch its own attn depending on whether it's supported or not - # In this case we get SDPA by default if it `_supports_Sdpa` else fallback to "eager" + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - - self.assertTrue( - model_sdpa.config._attn_implementation == {"encoder": encoder_attn, "decoder": decoder_attn} - ) - self.assertTrue(model_sdpa.config.encoder._attn_implementation == encoder_attn) - self.assertTrue(model_sdpa.config.decoder._attn_implementation == decoder_attn) + self.assertTrue(model_sdpa.config._attn_implementation == {"encoder": None, "decoder": None}) + self.assertTrue(model_sdpa.encoder.config._attn_implementation == encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) # Also test that nothing break if we request SDPA explicitly, when both sub-parts support it. - # If the model supports sdpa (i.e. one of sub-models supports it) we'll dispatch safely whenever possible - # Otherwise we should raise error that SDPA is not supported, as none of the sub-models support SDPA + # If the model supports sdpa (i.e. all of sub-models supports it) we'll dispatch safely + # Otherwise we should raise error that SDPA is not supported, as some of the sub-models doesn't support if encoder_attn == "sdpa" and decoder_attn == "sdpa": model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" @@ -452,8 +445,11 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_sdpa_explicit.config._attn_implementation == {"encoder": encoder_attn, "decoder": decoder_attn} ) - self.assertTrue(model_sdpa_explicit.config.encoder._attn_implementation == encoder_attn) - self.assertTrue(model_sdpa_explicit.config.decoder._attn_implementation == decoder_attn) + else: + with self.assertRaises(ValueError): + model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + ) model_eager = VisionEncoderDecoderModel.from_pretrained( tmpdirname, @@ -463,8 +459,8 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): model_eager = model_eager.eval().to(torch_device) self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) - self.assertTrue(model_eager.config.encoder._attn_implementation == "eager") - self.assertTrue(model_eager.config.decoder._attn_implementation == "eager") + self.assertTrue(model_eager.encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ @@ -603,7 +599,6 @@ def prepare_config_and_inputs(self): @require_torch class ViT2BertModelTest(EncoderDecoderMixin, unittest.TestCase): - has_attentions = True supports_sdpa = True # one submodel support SDPA def get_pretrained_model_and_inputs(self): @@ -758,7 +753,6 @@ def test_real_model_save_load_from_pretrained(self): @require_torch class ViT2TrOCR(EncoderDecoderMixin, unittest.TestCase): - has_attentions = True supports_sdpa = True # one submodel support SDPA def get_encoder_decoder_model(self, config, decoder_config): @@ -918,7 +912,6 @@ def test_real_model_save_load_from_pretrained(self): @require_torch class VIT2GPT2Test(EncoderDecoderMixin, unittest.TestCase): - has_attentions = True supports_sdpa = True # both submodels support SDPA def get_encoder_decoder_model(self, config, decoder_config): @@ -1036,7 +1029,6 @@ def test_real_model_save_load_from_pretrained(self): @require_torch class Donut2GPT2Test(EncoderDecoderMixin, unittest.TestCase): - has_attentions = True supports_sdpa = True # one submodel (GPT2) support SDPA def get_encoder_decoder_model(self, config, decoder_config): diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 12abfaf0963d..bef71891e370 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -184,7 +184,6 @@ class ModelTesterMixin: is_encoder_decoder = False has_attentions = True model_split_percents = [0.5, 0.7, 0.9] - is_multimodal = False def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = copy.deepcopy(inputs_dict) @@ -3712,8 +3711,9 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if (not self.is_multimodal and not self.all_model_classes[0]._supports_sdpa) or ( - self.is_multimodal and not self.supports_sdpa + print(not self.all_model_classes[0]._supports_sdpa, not self.all_model_classes[0]._is_composite) + if (not self.all_model_classes[0]._supports_sdpa and not self.all_model_classes[0]._is_composite) or ( + self.all_model_classes[0]._is_composite and not self.supports_sdpa ): self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") @@ -3777,20 +3777,18 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - if self.is_multimodal: - vision_supports_sdpa = ( - model.image_tower._supports_sdpa - if hasattr(model_sdpa, "image_tower") - else model.vision_tower._supports_sdpa - ) - vision_attn = "sdpa" if vision_supports_sdpa else "eager" + if model_sdpa._is_composite: + vision_model_name = "image_tower" if hasattr(model_sdpa, "image_tower") else "vision_tower" + vision_attn = "sdpa" if getattr(model, vision_model_name)._supports_sdpa else "eager" text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" - self.assertTrue(model_sdpa.config.vision_config._attn_implementation == vision_attn) - self.assertTrue(model_sdpa.config.text_config._attn_implementation == text_attn) + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) self.assertTrue( - model_sdpa.config._attn_implementation - == {"text_config": text_attn, "vision_config": vision_attn} + model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None} ) + self.assertTrue(model_sdpa.language_model.config._attn_implementation == text_attn) + self.assertTrue(getattr(model_sdpa, vision_model_name).config._attn_implementation == vision_attn) else: self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") @@ -3801,12 +3799,12 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - if self.is_multimodal: - self.assertTrue(model_eager.config.vision_config._attn_implementation == "eager") - self.assertTrue(model_eager.config.text_config._attn_implementation == "eager") + if model_eager._is_composite: self.assertTrue( model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} ) + self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, vision_model_name).config._attn_implementation == "eager") else: self.assertTrue(model_eager.config._attn_implementation == "eager") @@ -4044,8 +4042,8 @@ def test_sdpa_can_dispatch_on_flash(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if (not self.is_multimodal and not model_class._supports_sdpa) or ( - self.is_multimodal and not self.supports_sdpa + if (not model_class._is_composite and not model_class._supports_sdpa) or ( + model_class._is_composite and not self.supports_sdpa ): self.skipTest(f"{model_class.__name__} does not support SDPA") @@ -4092,8 +4090,8 @@ def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if (not self.is_multimodal and not model_class._supports_sdpa) or ( - self.is_multimodal and not self.supports_sdpa + if (not model_class._is_composite and not model_class._supports_sdpa) or ( + model_class._is_composite and not self.supports_sdpa ): self.skipTest(f"{model_class.__name__} does not support SDPA") @@ -4136,8 +4134,8 @@ def test_eager_matches_sdpa_generate(self): self.skipTest(f"{self.__class__.__name__} tests a model that does support generate: skipping this test") for model_class in self.all_generative_model_classes: - if (not self.is_multimodal and not model_class._supports_sdpa) or ( - self.is_multimodal and not self.supports_sdpa + if (not model_class._is_composite and not model_class._supports_sdpa) or ( + model_class._is_composite and not self.supports_sdpa ): self.skipTest(f"{model_class.__name__} does not support SDPA") From d1a291c07c65223fc00502bb880b1d861690b726 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 08:33:41 +0200 Subject: [PATCH 42/68] clean up --- docs/source/en/model_doc/idefics2.md | 17 ----------------- src/transformers/__init__.py | 8 ++------ src/transformers/modeling_utils.py | 16 ++++++++-------- .../models/auto/configuration_auto.py | 17 ----------------- src/transformers/models/auto/modeling_auto.py | 6 ------ .../models/blip_2/modeling_blip_2.py | 6 +++--- src/transformers/models/idefics2/__init__.py | 8 ++------ .../modeling_vision_text_dual_encoder.py | 4 ++-- src/transformers/utils/dummy_pt_objects.py | 14 -------------- tests/models/blip/test_modeling_blip.py | 6 +----- tests/models/idefics2/test_modeling_idefics2.py | 1 - .../instructblip/test_modeling_instructblip.py | 3 +-- .../test_modeling_instructblipvideo.py | 3 +-- tests/models/kosmos2/test_modeling_kosmos2.py | 3 +-- tests/models/llava/test_modeling_llava.py | 4 ++-- .../llava_next/test_modeling_llava_next.py | 4 ++-- .../test_modeling_llava_next_video.py | 4 ++-- .../models/paligemma/test_modeling_paligemma.py | 3 +-- .../video_llava/test_modeling_video_llava.py | 4 ++-- tests/models/vipllava/test_modeling_vipllava.py | 4 ++-- utils/check_copies.py | 2 -- utils/check_repo.py | 7 +++++-- utils/check_table.py | 6 ------ 23 files changed, 37 insertions(+), 113 deletions(-) diff --git a/docs/source/en/model_doc/idefics2.md b/docs/source/en/model_doc/idefics2.md index 3ba5bae786ce..5ad56b7b5c52 100644 --- a/docs/source/en/model_doc/idefics2.md +++ b/docs/source/en/model_doc/idefics2.md @@ -195,29 +195,12 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h [[autodoc]] Idefics2Config -## Idefics2VisionConfig - -[[autodoc]] Idefics2VisionConfig - -## Idefics2PerceiverConfig - -[[autodoc]] Idefics2PerceiverConfig - ## Idefics2Model [[autodoc]] Idefics2Model - forward -## Idefics2VisionTransformer - -[[autodoc]] Idefics2VisionTransformer - - forward - -## Idefics2PerceiverResampler - -[[autodoc]] Idefics2PerceiverResampler - - forward ## Idefics2ForConditionalGeneration diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index f24ee2cf5d96..9108367f35b3 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -472,7 +472,7 @@ "models.hubert": ["HubertConfig"], "models.ibert": ["IBertConfig"], "models.idefics": ["IdeficsConfig"], - "models.idefics2": ["Idefics2Config", "Idefics2PerceiverConfig", "Idefics2VisionConfig"], + "models.idefics2": ["Idefics2Config"], "models.imagegpt": ["ImageGPTConfig"], "models.informer": ["InformerConfig"], "models.instructblip": [ @@ -2344,10 +2344,8 @@ [ "Idefics2ForConditionalGeneration", "Idefics2Model", - "Idefics2PerceiverResampler", "Idefics2PreTrainedModel", "Idefics2Processor", - "Idefics2VisionTransformer", ] ) _import_structure["models.imagegpt"].extend( @@ -5154,7 +5152,7 @@ from .models.idefics import ( IdeficsConfig, ) - from .models.idefics2 import Idefics2Config, Idefics2PerceiverConfig, Idefics2VisionConfig + from .models.idefics2 import Idefics2Config from .models.imagegpt import ImageGPTConfig from .models.informer import InformerConfig from .models.instructblip import ( @@ -6873,10 +6871,8 @@ from .models.idefics2 import ( Idefics2ForConditionalGeneration, Idefics2Model, - Idefics2PerceiverResampler, Idefics2PreTrainedModel, Idefics2Processor, - Idefics2VisionTransformer, ) from .models.imagegpt import ( ImageGPTForCausalImageModeling, diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 0aeb91b61001..42a91706f632 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1519,14 +1519,16 @@ def _autoset_attn_implementation( # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available. requested_attn_implementation = config._attn_implementation_internal - # MultiModal-LLM/Encoder-Decoder related block: since they consist of two or might be more sub-configs - # we have to check and dispatch SDPA to each sub-config, in case any of them support it. - # If one sub-model supports SDPA while other doesn't, an error will be raised following the - # typical SDPA-dispatch path (i.e. if hard_check). Same goes for FA2. + # Composite models consisting of several PretrainedModels have to specify attention impl as a dict + # where keys are sub-config names. But most people will specify one `str` which means that should dispatch + # for all sub-models or do not specify anything (`None`). + # Below we check is a models is composite and manually prepare a dict of attn impl if not already passed as a dict. + # Later each sub-model will dispatch with its own attn impl, by calling `_from_config(attn_impl="sdpa/FA2/eager")` + # If any of sub-models don't support requested attn, an error will be raised sub_configs = { key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) } - if sub_configs: + if sub_configs: # so we have a composite model attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): attn_implementation_per_subconfig[key] = ( @@ -1535,7 +1537,7 @@ def _autoset_attn_implementation( else requested_attn_implementation.get(key) ) - if cls._is_composite: + if cls._is_composite: # some composite models don't use attn impl, e.g. VQ-VAE config._attn_implementation = attn_implementation_per_subconfig requested_attn_implementation = config._attn_implementation @@ -1638,8 +1640,6 @@ def _check_and_enable_flash_attn_2( If all checks pass and `hard_check_only` is False, the method will set the config attribute `attn_implementation` to "flash_attention_2" so that the model can initialize the correct attention module. """ - - # VLM/Encoder-Decoder etc. have to follow the sdpa attr of its sub-configs if not cls._supports_flash_attn_2: raise ValueError( f"{cls.__name__} does not support Flash Attention 2.0 yet. Please request to add support where" diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 5144fe795aad..8eb0daefca53 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -60,7 +60,6 @@ ("chinese_clip_vision_model", "ChineseCLIPVisionConfig"), ("clap", "ClapConfig"), ("clip", "CLIPConfig"), - ("clip_text_model", "CLIPTextConfig"), ("clip_vision_model", "CLIPVisionConfig"), ("clipseg", "CLIPSegConfig"), ("clvp", "ClvpConfig"), @@ -129,8 +128,6 @@ ("ibert", "IBertConfig"), ("idefics", "IdeficsConfig"), ("idefics2", "Idefics2Config"), - ("idefics2_perceiver_model", "Idefics2PerceiverConfig"), - ("idefics2_vision_model", "Idefics2VisionConfig"), ("imagegpt", "ImageGPTConfig"), ("informer", "InformerConfig"), ("instructblip", "InstructBlipConfig"), @@ -177,9 +174,7 @@ ("mra", "MraConfig"), ("mt5", "MT5Config"), ("musicgen", "MusicgenConfig"), - ("musicgen_decoder", "MusicgenDecoderConfig"), ("musicgen_melody", "MusicgenMelodyConfig"), - ("musicgen_melody_decoder", "MusicgenMelodyDecoderConfig"), ("mvp", "MvpConfig"), ("nat", "NatConfig"), ("nezha", "NezhaConfig"), @@ -235,7 +230,6 @@ ("sew", "SEWConfig"), ("sew-d", "SEWDConfig"), ("siglip", "SiglipConfig"), - ("siglip_text_model", "SiglipTextConfig"), ("siglip_vision_model", "SiglipVisionConfig"), ("speech-encoder-decoder", "SpeechEncoderDecoderConfig"), ("speech_to_text", "Speech2TextConfig"), @@ -341,7 +335,6 @@ ("chinese_clip_vision_model", "ChineseCLIPVisionModel"), ("clap", "CLAP"), ("clip", "CLIP"), - ("clip_text_model", "CLIPTextModel"), ("clip_vision_model", "CLIPVisionModel"), ("clipseg", "CLIPSeg"), ("clvp", "CLVP"), @@ -418,8 +411,6 @@ ("ibert", "I-BERT"), ("idefics", "IDEFICS"), ("idefics2", "Idefics2"), - ("idefics2_perceiver_model", "Idefics2PerceiverResampler"), - ("idefics2_vision_model", "Idefics2VisionTransformer"), ("imagegpt", "ImageGPT"), ("informer", "Informer"), ("instructblip", "InstructBLIP"), @@ -475,9 +466,7 @@ ("mra", "MRA"), ("mt5", "MT5"), ("musicgen", "MusicGen"), - ("musicgen_decoder", "MusicGenDecoder"), ("musicgen_melody", "MusicGen Melody"), - ("musicgen_melody_decoder", "MusicGen Melody Decoder"), ("mvp", "MVP"), ("nat", "NAT"), ("nezha", "Nezha"), @@ -535,7 +524,6 @@ ("sew", "SEW"), ("sew-d", "SEW-D"), ("siglip", "SigLIP"), - ("siglip_text_model", "SiglipTextModel"), ("siglip_vision_model", "SiglipVisionModel"), ("speech-encoder-decoder", "Speech Encoder decoder"), ("speech_to_text", "Speech2Text"), @@ -648,17 +636,12 @@ ("donut-swin", "donut"), ("kosmos-2", "kosmos2"), ("maskformer-swin", "maskformer"), - ("musicgen_decoder", "musicgen"), ("musicgen_melody_decoder", "musicgen_melody"), ("xclip", "x_clip"), - ("clip_text_model", "clip"), ("clip_vision_model", "clip"), - ("siglip_text_model", "siglip"), ("siglip_vision_model", "siglip"), ("chinese_clip_vision_model", "chinese_clip"), ("rt_detr_resnet", "rt_detr"), - ("idefics2_vision_model", "idefics2"), - ("idefics2_perceiver_model", "idefics2"), ] ) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 1ff6b96306fd..d096abf43426 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -60,7 +60,6 @@ ("chinese_clip_vision_model", "ChineseCLIPVisionModel"), ("clap", "ClapModel"), ("clip", "CLIPModel"), - ("clip_text_model", "CLIPTextModel"), ("clip_vision_model", "CLIPVisionModel"), ("clipseg", "CLIPSegModel"), ("clvp", "ClvpModelForConditionalGeneration"), @@ -126,8 +125,6 @@ ("ibert", "IBertModel"), ("idefics", "IdeficsModel"), ("idefics2", "Idefics2Model"), - ("idefics2_perceiver_model", "Idefics2PerceiverResampler"), - ("idefics2_vision_model", "Idefics2VisionTransformer"), ("imagegpt", "ImageGPTModel"), ("informer", "InformerModel"), ("jamba", "JambaModel"), @@ -169,9 +166,7 @@ ("mra", "MraModel"), ("mt5", "MT5Model"), ("musicgen", "MusicgenModel"), - ("musicgen_decoder", "MusicgenModel"), ("musicgen_melody", "MusicgenMelodyModel"), - ("musicgen_melody_decoder", "MusicgenMelodyModel"), ("mvp", "MvpModel"), ("nat", "NatModel"), ("nezha", "NezhaModel"), @@ -220,7 +215,6 @@ ("sew", "SEWModel"), ("sew-d", "SEWDModel"), ("siglip", "SiglipModel"), - ("siglip_text_model", "SiglipTextModel"), ("siglip_vision_model", "SiglipVisionModel"), ("speech_to_text", "Speech2TextModel"), ("speecht5", "SpeechT5Model"), diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index aa60ce9be272..6b5e355a5ffa 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1600,7 +1600,7 @@ def __init__(self, config: Blip2Config): super().__init__(config) self.vision_model = Blip2VisionModel._from_config( - config.vision_config, attn_implementation=config.vision_config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) @@ -1609,11 +1609,11 @@ def __init__(self, config: Blip2Config): self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) else: language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config.text_config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) # Update _tied_weights_keys using the base model used. diff --git a/src/transformers/models/idefics2/__init__.py b/src/transformers/models/idefics2/__init__.py index 13f350150196..1d8d3e4b571d 100644 --- a/src/transformers/models/idefics2/__init__.py +++ b/src/transformers/models/idefics2/__init__.py @@ -16,7 +16,7 @@ from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available -_import_structure = {"configuration_idefics2": ["Idefics2Config", "Idefics2VisionConfig", "Idefics2PerceiverConfig"]} +_import_structure = {"configuration_idefics2": ["Idefics2Config"]} try: @@ -38,13 +38,11 @@ "Idefics2ForConditionalGeneration", "Idefics2PreTrainedModel", "Idefics2Model", - "Idefics2VisionTransformer", - "Idefics2PerceiverResampler", ] _import_structure["processing_idefics2"] = ["Idefics2Processor"] if TYPE_CHECKING: - from .configuration_idefics2 import Idefics2Config, Idefics2PerceiverConfig, Idefics2VisionConfig + from .configuration_idefics2 import Idefics2Config try: if not is_vision_available(): @@ -63,9 +61,7 @@ from .modeling_idefics2 import ( Idefics2ForConditionalGeneration, Idefics2Model, - Idefics2PerceiverResampler, Idefics2PreTrainedModel, - Idefics2VisionTransformer, ) from .processing_idefics2 import Idefics2Processor diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 4a4e68d58579..5babc62ef649 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -199,8 +199,8 @@ def __init__( # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced - self.config.vision_config._attn_implementation = self.vision_config.config._attn_implementation - self.config.text_config._attn_implementation = self.text_config.config._attn_implementation + self.config.vision_config._attn_implementation = self.vision_model.config._attn_implementation + self.config.text_config._attn_implementation = self.text_model.config._attn_implementation self.vision_model.config = self.config.vision_config self.text_model.config = self.config.text_config diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 24ce57578917..de739c6e7004 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -4775,13 +4775,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) -class Idefics2PerceiverResampler(metaclass=DummyObject): - _backends = ["torch"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["torch"]) - - class Idefics2PreTrainedModel(metaclass=DummyObject): _backends = ["torch"] @@ -4796,13 +4789,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) -class Idefics2VisionTransformer(metaclass=DummyObject): - _backends = ["torch"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["torch"]) - - class ImageGPTForCausalImageModeling(metaclass=DummyObject): _backends = ["torch"] diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 981f4523c7e9..b36059933c4d 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -446,7 +446,6 @@ class BlipModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False - is_multimodal = True # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA @@ -811,7 +810,6 @@ class BlipVQAModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False test_torchscript = False - is_multimodal = True # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA @@ -897,7 +895,6 @@ class BlipTextRetrievalModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False test_torchscript = False - is_multimodal = True # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA @@ -1131,8 +1128,7 @@ class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False test_torchscript = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA supports_sdpa = False diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 24c2dcdf808e..e3dd778273f0 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -408,7 +408,6 @@ class Idefics2ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTest test_head_masking = False test_torchscript = False - is_multimodal = True # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because LM/vision models used in tests don't support SDPA diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 939bbb3d001a..2e7db8af9c1b 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -461,8 +461,7 @@ class InstructBlipForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, Gene test_attention_outputs = False test_torchscript = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA supports_sdpa = False diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index c1d0d5113c54..1f29a490f19a 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -482,8 +482,7 @@ class InstructBlipVideoForConditionalGenerationDecoderOnlyTest( test_attention_outputs = False test_torchscript = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because vision models used in tests don't support SDPA supports_sdpa = False diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index c7aa4d9aca48..7d97d8faa87c 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -258,8 +258,7 @@ class Kosmos2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_resize_embeddings = False test_attention_outputs = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to False because LM/vision models used in tests don't support SDPA supports_sdpa = False diff --git a/tests/models/llava/test_modeling_llava.py b/tests/models/llava/test_modeling_llava.py index 07ebcb81c961..a68b08c83bf9 100644 --- a/tests/models/llava/test_modeling_llava.py +++ b/tests/models/llava/test_modeling_llava.py @@ -182,8 +182,8 @@ class LlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase pipeline_model_mapping = {"image-to-text": LlavaForConditionalGeneration} if is_torch_available() else {} test_pruning = False test_head_masking = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA supports_sdpa = True diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index e1d71f2704dc..f0ba1a3b92f7 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -216,8 +216,8 @@ class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, unittest.Test all_generative_model_classes = (LlavaNextForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA supports_sdpa = True diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 887124ee3c2f..19eb37342c47 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -231,8 +231,8 @@ class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, unittest all_generative_model_classes = (LlavaNextVideoForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA supports_sdpa = True diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index beb276402685..70761a91b969 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -182,8 +182,7 @@ class PaliGemmaForConditionalGenerationModelTest(ModelTesterMixin, unittest.Test test_torchscript = False test_head_masking = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA supports_sdpa = True diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index d27c4cfe39cf..c9ceb08f1375 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -200,8 +200,8 @@ class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.Tes test_pruning = False test_resize_embeddings = True test_head_masking = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA supports_sdpa = True diff --git a/tests/models/vipllava/test_modeling_vipllava.py b/tests/models/vipllava/test_modeling_vipllava.py index 63e9a2337bd1..9ada5736e69a 100644 --- a/tests/models/vipllava/test_modeling_vipllava.py +++ b/tests/models/vipllava/test_modeling_vipllava.py @@ -163,8 +163,8 @@ class VipLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestC test_pruning = False test_resize_embeddings = True test_head_masking = False - is_multimodal = True - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used + + # We define this flag here because in VLMs these flags depend on which LM/vision models are used # So we can't know if SDPA is supported before starting to load the model # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA supports_sdpa = True diff --git a/utils/check_copies.py b/utils/check_copies.py index 77da83a51787..4bb5c6fef4ee 100644 --- a/utils/check_copies.py +++ b/utils/check_copies.py @@ -1086,9 +1086,7 @@ def _find_text_in_file(filename: str, start_prompt: str, end_prompt: str) -> Tup "Vision Encoder decoder", "VisionTextDualEncoder", "CLIPVisionModel", - "CLIPTextModel", "SiglipVisionModel", - "SiglipTextModel", "ChineseCLIPVisionModel", ] diff --git a/utils/check_repo.py b/utils/check_repo.py index 938cd3e761ff..d53bfcb94305 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -81,6 +81,8 @@ "SeamlessM4Tv2TextToUnitModel", "SeamlessM4Tv2CodeHifiGan", "SeamlessM4Tv2TextToUnitForConditionalGeneration", + "Idefics2PerceiverResampler", + "Idefics2VisionTransformer", ] # Update this list for models that are not tested with a comment explaining the reason it should not be. @@ -128,8 +130,6 @@ "SeamlessM4TCodeHifiGan", # Building part of bigger (tested) model. "SeamlessM4TTextToUnitForConditionalGeneration", # Building part of bigger (tested) model. "ChameleonVQVAE", # VQVAE here is used only for encoding (discretizing) and is tested as part of bigger model - "Idefics2VisionTransformer", # Idefics2 modules are tested as part of a bigger model - "Idefics2PerceiverResampler", # Idefics2 modules are tested as part of a bigger model ] # Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't @@ -319,7 +319,10 @@ "SeamlessM4Tv2CodeHifiGan", "SeamlessM4Tv2ForSpeechToSpeech", # no auto class for speech-to-speech "SegGptForImageSegmentation", + "SiglipVisionModel", + "SiglipTextModel", "ChameleonVQVAE", # no autoclass for VQ-VAE models + "CLIPTextModel", ] # DO NOT edit this list! diff --git a/utils/check_table.py b/utils/check_table.py index d3aeef78e1b1..5cb0f8c3d2e5 100644 --- a/utils/check_table.py +++ b/utils/check_table.py @@ -175,14 +175,8 @@ def _center_text(text: str, width: int) -> str: } MODEL_NAMES_TO_IGNORE = [ "CLIPVisionModel", - "CLIPTextModel", - "Idefics2VisionTransformer", - "Idefics2PerceiverResampler", "SiglipVisionModel", - "SiglipTextModel", "ChineseCLIPVisionModel", - "MusicGenDecoder", - "MusicGen Melody Decoder", ] From 57119b11eee8e836271643b08990069688563d83 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 09:09:24 +0200 Subject: [PATCH 43/68] add FA2 test --- .../models/musicgen/modeling_musicgen.py | 1 - .../modeling_musicgen_melody.py | 1 - tests/test_modeling_common.py | 54 ++++++++++++++++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/musicgen/modeling_musicgen.py b/src/transformers/models/musicgen/modeling_musicgen.py index cb9086b93892..ee5c36f35023 100644 --- a/src/transformers/models/musicgen/modeling_musicgen.py +++ b/src/transformers/models/musicgen/modeling_musicgen.py @@ -704,7 +704,6 @@ class MusicgenPreTrainedModel(PreTrainedModel): _no_split_modules = ["MusicgenDecoderLayer", "MusicgenAttention"] _supports_flash_attn_2 = True _supports_sdpa = True - _is_composite = True def _init_weights(self, module): std = self.config.initializer_factor diff --git a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py index a1ce7b3f24b2..359a608ea649 100644 --- a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py +++ b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py @@ -663,7 +663,6 @@ class MusicgenMelodyPreTrainedModel(PreTrainedModel): _no_split_modules = ["MusicgenMelodyDecoderLayer", "MusicgenMelodyAttention"] _supports_flash_attn_2 = True _supports_sdpa = True - _is_composite = True def _init_weights(self, module): std = self.config.initializer_factor diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index bef71891e370..e5dfaac12584 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3711,7 +3711,6 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - print(not self.all_model_classes[0]._supports_sdpa, not self.all_model_classes[0]._is_composite) if (not self.all_model_classes[0]._supports_sdpa and not self.all_model_classes[0]._is_composite) or ( self.all_model_classes[0]._is_composite and not self.supports_sdpa ): @@ -4301,6 +4300,59 @@ def test_flash_attn_2_generate_use_cache(self): use_cache=True, ) + @require_flash_attn + @require_torch_gpu + @mark.flash_attn_test + @slow + def test_flash_attn_2_can_dispatch_composite_models(self): + """ + Tests if composite models can dispatch on FA2 if the sub-models supports FA2. + The tests is needed as we handle differently composite models and we cannot check them + with above tests. If any of the sub-models does not support FA2, we'll raise an error when dispatching + that particular sub-model. Otherwise we dispatch safely in all sub-modules. + """ + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not is_torch_fp16_available_on_device(torch_device): + self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") + + torch_dtype = torch.float16 + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + if not model_class._is_composite: + self.skipTest("This model is not a composte model!") + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + + supports_fa2_all_modules = all( + module._supports_flash_attn_2 + for name, module in model.named_modules() + if isinstance(module, PreTrainedModel) and name != "" + ) + if not supports_fa2_all_modules: + with self.assertRaises(ValueError): + model_fa2 = model_class.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="flash_attention_2" + ) + else: + model_fa2 = model_class.from_pretrained( + tmpdirname, torch_dtype=torch_dtype, attn_implementation="flash_attention_2" + ) + self.assertTrue("flash_attention_2" in model_fa2.config._attn_implementation.values()) + + has_fa2 = False + for name, submodule in model_fa2.named_modules(): + class_name = submodule.__class__.__name__ + if "FlashAttention" in class_name: + has_fa2 = True + break + if not has_fa2: + raise ValueError("The FA2 model should have FA2 layers") + @require_flash_attn @require_torch_gpu @require_bitsandbytes From cfb91984af66d9c5654a523a74eead57ea14a8d7 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 10:09:50 +0200 Subject: [PATCH 44/68] more tests --- tests/test_modeling_common.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index e5dfaac12584..745899ad66f4 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4577,6 +4577,41 @@ def test_flash_attn_2_from_config(self): self.assertFalse(fa2_correctly_converted) + def test_attn_implementation_composite_models(self): + """ + Tests if composite models can receive a dict object as attn_implementation, where each key should be + one of the sub-configs from the model's config. + """ + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + for model_class in self.all_model_classes: + if not model_class._is_composite: + self.skipTest("Model is not a composite model.") + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + sub_configs = { + key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) + } + + # set eager as it will be the one supported in all models, for the sake of testing + # if passing a dicr fails or not + attn_implementation_per_subconfig = {} + for key, sub_config in sub_configs.items(): + attn_implementation_per_subconfig[key] = "eager" + + config._attn_implementation = attn_implementation_per_subconfig + model = model_class(config) + self.assertTrue(model.config._attn_implementation == attn_implementation_per_subconfig) + for name, submodule in model.named_modules(): + class_name = submodule.__class__.__name__ + if ( + "SdpaAttention" in class_name + or "SdpaSelfAttention" in class_name + or "FlashAttention" in class_name + ): + raise ValueError("The eager model should not have SDPA/FA2 attention layers") + def _get_custom_4d_mask_test_data(self): # Sequence in which all but the last token is the same input_ids = torch.tensor( From a2e90627f6720e5a80af69b5b69aeeb8342473c5 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 10:12:31 +0200 Subject: [PATCH 45/68] style --- tests/models/idefics2/test_modeling_idefics2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index a42e91c8b56a..1e03d6b813a6 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -35,8 +35,8 @@ require_bitsandbytes, require_flash_attn, require_torch, - require_torch_sdpa, require_torch_gpu, + require_torch_sdpa, slow, torch_device, ) From e72c03df5b0239eb42d3514c4d009570aca94a79 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 10:40:49 +0200 Subject: [PATCH 46/68] skip tests --- tests/models/musicgen/test_modeling_musicgen.py | 4 ++++ tests/models/openai/test_modeling_openai.py | 4 ++++ tests/models/xglm/test_modeling_xglm.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index def4bb5e1199..d7c3fa4d0fdf 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -270,6 +270,10 @@ def test_model_get_set_embeddings(self): def test_inputs_embeds_matches_input_ids(self): pass + @unittest.skip(reason="MusicGen does not use inputs_embeds") + def test_inputs_embeds_matches_input_ids_with_generate(self): + pass + @unittest.skip(reason="MusicGen does not support all arguments tested") def test_model_outputs_equivalence(self): pass diff --git a/tests/models/openai/test_modeling_openai.py b/tests/models/openai/test_modeling_openai.py index 49e6d50bc428..c15f2ae430ff 100644 --- a/tests/models/openai/test_modeling_openai.py +++ b/tests/models/openai/test_modeling_openai.py @@ -273,6 +273,10 @@ def test_model_from_pretrained(self): model = OpenAIGPTModel.from_pretrained(model_name) self.assertIsNotNone(model) + @unittest.skip(reason="OpenAIGPT does not accept inputs_embeds in generation") + def test_inputs_embeds_matches_input_ids_with_generate(self): + pass + @require_torch class OPENAIGPTModelLanguageGenerationTest(unittest.TestCase): diff --git a/tests/models/xglm/test_modeling_xglm.py b/tests/models/xglm/test_modeling_xglm.py index a9db8db6e0ad..e0b03fe685f2 100644 --- a/tests/models/xglm/test_modeling_xglm.py +++ b/tests/models/xglm/test_modeling_xglm.py @@ -357,6 +357,10 @@ def test_model_from_pretrained(self): def test_model_parallelism(self): super().test_model_parallelism() + @unittest.skip(reason="XGLM does not accept inputs_embeds in generation") + def test_inputs_embeds_matches_input_ids_with_generate(self): + pass + @require_torch class XGLMModelLanguageGenerationTest(unittest.TestCase): From 60df87202ad069ae05d316d7f737ee2256964c73 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 11:29:42 +0200 Subject: [PATCH 47/68] why did these started failing now? --- tests/models/mamba2/test_modeling_mamba2.py | 4 ++++ tests/models/openai/test_modeling_openai.py | 4 ---- .../recurrent_gemma/test_modeling_recurrent_gemma.py | 6 ++++++ tests/models/xglm/test_modeling_xglm.py | 4 ---- tests/test_modeling_common.py | 7 ++++--- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/models/mamba2/test_modeling_mamba2.py b/tests/models/mamba2/test_modeling_mamba2.py index 13cc22561fe1..bbd8e2834196 100644 --- a/tests/models/mamba2/test_modeling_mamba2.py +++ b/tests/models/mamba2/test_modeling_mamba2.py @@ -211,6 +211,10 @@ def test_beam_sample_generate(self): def test_generate_without_input_ids(self): pass + @unittest.skip(reason="To fix, Mamba 2 cache slicing test case is an edge case") + def test_inputs_embeds_matches_input_ids_with_generate(self): + pass + @unittest.skip(reason="To fix, Mamba 2 cache slicing test case is an edge case") def test_greedy_generate_dict_outputs_use_cache(self): pass diff --git a/tests/models/openai/test_modeling_openai.py b/tests/models/openai/test_modeling_openai.py index c15f2ae430ff..49e6d50bc428 100644 --- a/tests/models/openai/test_modeling_openai.py +++ b/tests/models/openai/test_modeling_openai.py @@ -273,10 +273,6 @@ def test_model_from_pretrained(self): model = OpenAIGPTModel.from_pretrained(model_name) self.assertIsNotNone(model) - @unittest.skip(reason="OpenAIGPT does not accept inputs_embeds in generation") - def test_inputs_embeds_matches_input_ids_with_generate(self): - pass - @require_torch class OPENAIGPTModelLanguageGenerationTest(unittest.TestCase): diff --git a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py index ad542db2733b..9310a505ab65 100644 --- a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py +++ b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py @@ -390,6 +390,12 @@ def test_left_padding_compatibility(self): def test_assisted_decoding_sample(self): pass + @unittest.skip( + reason="RecurentGemma generation tests are not fully supported" + ) # TODO: @gante after adding MixinTests + def test_inputs_embeds_matches_input_ids_with_generate(self): + pass + def _check_hidden_states_for_generate( self, batch_size, hidden_states, min_length, max_length, config, use_cache=False, num_beam_groups=1 ): diff --git a/tests/models/xglm/test_modeling_xglm.py b/tests/models/xglm/test_modeling_xglm.py index e0b03fe685f2..a9db8db6e0ad 100644 --- a/tests/models/xglm/test_modeling_xglm.py +++ b/tests/models/xglm/test_modeling_xglm.py @@ -357,10 +357,6 @@ def test_model_from_pretrained(self): def test_model_parallelism(self): super().test_model_parallelism() - @unittest.skip(reason="XGLM does not accept inputs_embeds in generation") - def test_inputs_embeds_matches_input_ids_with_generate(self): - pass - @require_torch class XGLMModelLanguageGenerationTest(unittest.TestCase): diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 9de69cd96bae..4efea69b9c66 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -2824,13 +2824,14 @@ def test_inputs_embeds_matches_input_ids_with_generate(self): for model_class in self.all_model_classes: if model_class.__name__ not in get_values(MODEL_FOR_CAUSAL_LM_MAPPING_NAMES): continue + model = model_class(config) model.to(torch_device) model.eval() - model_forward_args = inspect.signature(model.forward).parameters - if "inputs_embeds" not in model_forward_args: - self.skipTest(reason="This model doesn't use `inputs_embeds`") + model_generation_args = inspect.signature(model.prepare_inputs_for_generation).parameters + if "inputs_embeds" not in model_generation_args: + self.skipTest(reason="This model doesn't use `inputs_embeds` for generation") inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) pad_token_id = config.pad_token_id if config.pad_token_id is not None else 1 From 6d128971b362e51f7697777224acaf4702274834 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 11:33:17 +0200 Subject: [PATCH 48/68] no attributes for FA2 needed --- src/transformers/models/llava/modeling_llava.py | 1 - src/transformers/models/llava_next/modeling_llava_next.py | 1 - .../models/llava_next_video/modeling_llava_next_video.py | 1 - src/transformers/models/paligemma/modeling_paligemma.py | 1 - src/transformers/models/video_llava/modeling_video_llava.py | 1 - src/transformers/models/vipllava/modeling_vipllava.py | 1 - 6 files changed, 6 deletions(-) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 03eef995e864..bc6958476884 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -128,7 +128,6 @@ class LlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True _is_composite = True _supports_cache_class = True diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 0d101eb34934..472f4d58b641 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -231,7 +231,6 @@ class LlavaNextPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaNextVisionAttention"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True _is_composite = True _supports_cache_class = True diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index e949228f5c10..11b7eae9104f 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -271,7 +271,6 @@ class LlavaNextVideoPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaNextVideoVisionAttention"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True _is_composite = True _supports_cache_class = True diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 35072043419b..05f7240e04b4 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -125,7 +125,6 @@ class PaliGemmaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["PaliGemmaMultiModalProjector"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = False _is_composite = True _supports_cache_class = True diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 91c12d679cc7..5dcff367d027 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -125,7 +125,6 @@ class VideoLlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["VideoLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True _is_composite = True _supports_cache_class = True diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index bd90c1c2261e..00fed185e23e 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -134,7 +134,6 @@ class VipLlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["VipLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True _is_composite = True _supports_cache_class = True From 957e64eb344055224429d85bd23c7e6e2e44b38a Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 9 Aug 2024 11:38:20 +0200 Subject: [PATCH 49/68] one tiny test --- tests/test_modeling_common.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 4efea69b9c66..943cf5752fc7 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4211,7 +4211,20 @@ def test_eager_matches_sdpa_generate(self): low_cpu_mem_usage=True, ).to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + if model_sdpa._is_composite: + vision_model_name = "image_tower" if hasattr(model_sdpa, "image_tower") else "vision_tower" + vision_attn = "sdpa" if getattr(model, vision_model_name)._supports_sdpa else "eager" + text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue( + model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None} + ) + self.assertTrue(model_sdpa.language_model.config._attn_implementation == text_attn) + self.assertTrue(getattr(model_sdpa, vision_model_name).config._attn_implementation == vision_attn) + else: + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") model_eager = model_class.from_pretrained( tmpdirname, @@ -4220,7 +4233,14 @@ def test_eager_matches_sdpa_generate(self): attn_implementation="eager", ).to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") + if model_eager._is_composite: + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} + ) + self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, vision_model_name).config._attn_implementation == "eager") + else: + self.assertTrue(model_eager.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ From 94e7578a7da87bfe6ea58750143c81173d0f8c1b Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 18 Sep 2024 17:44:33 +0200 Subject: [PATCH 50/68] address comment about FA2 false warning --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index d1a030cd6b95..37cf0e99ba32 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1444,7 +1444,7 @@ def _from_config(cls, config, **kwargs): torch_dtype (`torch.dtype`, *optional*): Override the default `torch.dtype` and load the model under this dtype. """ - torch_dtype = kwargs.pop("torch_dtype", None) + torch_dtype = kwargs.pop("torch_dtype", torch.get_default_dtype()) use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False) # override default dtype if needed From 59aa48049bad812fcddbae4630f140ce8fb0308d Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 3 Oct 2024 12:25:46 +0200 Subject: [PATCH 51/68] style --- tests/models/idefics2/test_modeling_idefics2.py | 2 +- tests/models/mamba2/test_modeling_mamba2.py | 6 ------ .../models/recurrent_gemma/test_modeling_recurrent_gemma.py | 4 ---- tests/test_modeling_common.py | 2 +- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index d52318f5b544..3cfef5baa02a 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -36,8 +36,8 @@ require_flash_attn, require_torch, require_torch_gpu, - require_torch_sdpa, require_torch_multi_gpu, + require_torch_sdpa, slow, torch_device, ) diff --git a/tests/models/mamba2/test_modeling_mamba2.py b/tests/models/mamba2/test_modeling_mamba2.py index fa944f5afb96..c6488d473c0d 100644 --- a/tests/models/mamba2/test_modeling_mamba2.py +++ b/tests/models/mamba2/test_modeling_mamba2.py @@ -279,12 +279,6 @@ def recursive_check(tuple_object, dict_object): dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) check_equivalence(model, tuple_inputs, dict_inputs, {"output_hidden_states": True}) - @unittest.skip( - reason="Mamba2 does not support generating with input embeddings (custom cache_position computation)" - ) - def test_inputs_embeds_matches_input_ids_with_generate(self): - pass - @require_torch @slow diff --git a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py index 90a6b10f0d04..f2c6c1ddb1c3 100644 --- a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py +++ b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py @@ -419,10 +419,6 @@ def _check_hidden_states_for_generate( def test_initialization(self): pass - @unittest.skip(reason="RecurrentGemma does not support generating with input embeddings (missing position_ids)") - def test_inputs_embeds_matches_input_ids_with_generate(self): - pass - @require_torch_accelerator @slow diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 88f4a65e067b..025299e2a62a 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4458,7 +4458,7 @@ def test_flash_attn_2_can_dispatch_composite_models(self): break if not has_fa2: raise ValueError("The FA2 model should have FA2 layers") - + @require_flash_attn @require_torch_gpu @mark.flash_attn_test From 1dc8bd185c4891ac93d69ab1aa2a7240d0d5609f Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 3 Oct 2024 17:36:19 +0200 Subject: [PATCH 52/68] add new models and resolve conflicts --- src/transformers/modeling_utils.py | 12 +- .../models/blip_2/modeling_blip_2.py | 6 - src/transformers/models/clip/modeling_clip.py | 8 +- .../modeling_encoder_decoder.py | 1 - .../models/idefics2/modeling_idefics2.py | 13 +- .../models/idefics3/modeling_idefics3.py | 8 +- .../instructblip/modeling_instructblip.py | 5 - .../modeling_instructblipvideo.py | 6 - .../models/llava/modeling_llava.py | 1 - .../models/llava_next/modeling_llava_next.py | 1 - .../modeling_llava_next_video.py | 1 - .../modeling_llava_onevision.py | 4 +- .../models/mllama/modeling_mllama.py | 6 +- .../models/musicgen/modeling_musicgen.py | 1 - .../modeling_musicgen_melody.py | 1 - .../models/paligemma/modeling_paligemma.py | 1 - .../qwen2_audio/modeling_qwen2_audio.py | 15 +- .../models/qwen2_vl/configuration_qwen2_vl.py | 28 ++ .../models/qwen2_vl/modeling_qwen2_vl.py | 17 +- .../models/siglip/modeling_siglip.py | 6 +- .../modeling_speech_encoder_decoder.py | 1 - .../video_llava/modeling_video_llava.py | 1 - .../models/vipllava/modeling_vipllava.py | 3 +- .../modeling_vision_encoder_decoder.py | 1 - .../modeling_vision_text_dual_encoder.py | 1 - tests/models/blip/test_modeling_blip.py | 30 +- tests/models/blip_2/test_modeling_blip_2.py | 20 +- tests/models/clip/test_modeling_clip.py | 108 +++++--- tests/models/gemma2/test_modeling_gemma2.py | 5 + tests/models/idefics/test_modeling_idefics.py | 65 +---- .../models/idefics2/test_modeling_idefics2.py | 48 +--- .../test_modeling_instructblip.py | 10 +- .../test_modeling_instructblipvideo.py | 10 +- tests/models/kosmos2/test_modeling_kosmos2.py | 10 +- tests/models/llava/test_modeling_llava.py | 6 +- .../llava_next/test_modeling_llava_next.py | 6 +- .../test_modeling_llava_next_video.py | 6 +- .../test_modeling_llava_onevision.py | 1 + tests/models/mllama/test_modeling_mllama.py | 1 + .../models/musicgen/test_modeling_musicgen.py | 112 ++++---- .../test_modeling_musicgen_melody.py | 112 ++++---- .../paligemma/test_modeling_paligemma.py | 6 +- .../qwen2_audio/test_modeling_qwen2_audio.py | 54 ++++ .../models/qwen2_vl/test_modeling_qwen2_vl.py | 1 + tests/models/siglip/test_modeling_siglip.py | 107 +++++--- .../video_llava/test_modeling_video_llava.py | 6 +- .../models/vipllava/test_modeling_vipllava.py | 6 +- tests/test_modeling_common.py | 258 +++++++++--------- 48 files changed, 588 insertions(+), 548 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e2007ddf181b..c25515aa6456 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1385,9 +1385,6 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix # SDPA support _supports_sdpa = False - # Composite models consisting of several PretrainedModels - _is_composite = False - # Has support for a `Cache` instance as `past_key_values`? Does it support a `StaticCache`? _supports_cache_class = False _supports_static_cache = False @@ -1592,7 +1589,9 @@ def _autoset_attn_implementation( sub_configs = { key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) } - if sub_configs: # so we have a composite model + if sub_configs and all( + name not in cls.__name__.lower() for name in ["chameleon", "dbrx"] + ): # so we have a composite model attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): attn_implementation_per_subconfig[key] = ( @@ -1601,9 +1600,8 @@ def _autoset_attn_implementation( else requested_attn_implementation.get(key) ) - if cls._is_composite: # some composite models don't use attn impl, e.g. VQ-VAE - config._attn_implementation = attn_implementation_per_subconfig - requested_attn_implementation = config._attn_implementation + config._attn_implementation = attn_implementation_per_subconfig + requested_attn_implementation = config._attn_implementation if use_flash_attention_2: logger.warning_once( diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 8500374a6ff3..496695da064d 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -410,7 +410,6 @@ class Blip2PreTrainedModel(PreTrainedModel): config_class = Blip2Config base_model_prefix = "blip" supports_gradient_checkpointing = True - _is_composite = True _no_split_modules = [ "Blip2Attention", @@ -712,9 +711,6 @@ class Blip2VisionModel(Blip2PreTrainedModel): main_input_name = "pixel_values" config_class = Blip2VisionConfig - # Ignore copy - _is_composite = False - def __init__(self, config: Blip2VisionConfig): super().__init__(config) self.config = config @@ -1252,8 +1248,6 @@ class Blip2QFormerModel(Blip2PreTrainedModel): Querying Transformer (Q-Former), used in BLIP-2. """ - _is_composite = False - def __init__(self, config: Blip2QFormerConfig): super().__init__(config) self.config = config diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 7b93079cb9ad..6562683f38aa 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -637,7 +637,6 @@ class CLIPPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _supports_sdpa = True _supports_flash_attn_2 = True - _is_composite = True def _init_weights(self, module): """Initialize the weights""" @@ -1003,7 +1002,6 @@ def forward( ) class CLIPTextModel(CLIPPreTrainedModel): config_class = CLIPTextConfig - _is_composite = False _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer"] @@ -1126,7 +1124,6 @@ class CLIPVisionModel(CLIPPreTrainedModel): config_class = CLIPVisionConfig main_input_name = "pixel_values" _no_split_modules = ["CLIPEncoderLayer"] - _is_composite = False def __init__(self, config: CLIPVisionConfig): super().__init__(config) @@ -1428,7 +1425,6 @@ def forward( ) class CLIPTextModelWithProjection(CLIPPreTrainedModel): config_class = CLIPTextConfig - _is_composite = False _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer"] @@ -1512,7 +1508,6 @@ def forward( class CLIPVisionModelWithProjection(CLIPPreTrainedModel): config_class = CLIPVisionConfig main_input_name = "pixel_values" - _is_composite = False def __init__(self, config: CLIPVisionConfig): super().__init__(config) @@ -1594,14 +1589,13 @@ def forward( ) class CLIPForImageClassification(CLIPPreTrainedModel): main_input_name = "pixel_values" - _is_composite = False def __init__(self, config: CLIPConfig) -> None: super().__init__(config) self.num_labels = config.num_labels vision_model = CLIPVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.vision_model = vision_model.vision_model diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index 93488c27393d..c56110967e4b 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -179,7 +179,6 @@ class EncoderDecoderModel(PreTrainedModel): main_input_name = "input_ids" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False - _is_composite = True def __init__( self, diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 0a1a38d8e681..4d9819cb18ac 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -589,6 +589,7 @@ def forward( [`~PreTrainedModel.from_pretrained`] method to load the model weights. """ + @add_start_docstrings( "The bare Idefics2 Model outputting raw hidden-states without any specific head on top.", IDEFICS2_START_DOCSTRING, @@ -999,10 +1000,15 @@ def __init__(self, config, layer_idx: int): self.n_latents = config.perceiver_config.resampler_n_latents self.depth = config.perceiver_config.resampler_depth self.rms_norm_eps = config.text_config.rms_norm_eps + attn_implementation = ( + config._attn_implementation["perceiver_config"] + if config._attn_implementation["perceiver_config"] is not None + else "eager" + ) self.input_latents_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.input_context_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) - self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx) + self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[attn_implementation](config, layer_idx=layer_idx) self.post_attention_layernorm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.mlp = Idefics2MLP( hidden_size=config.text_config.hidden_size, @@ -1086,7 +1092,6 @@ def forward( ) class Idefics2PerceiverResampler(Idefics2PreTrainedModel): _supports_sdpa = False - _is_composite = False def __init__(self, config) -> None: super().__init__(config) @@ -1151,9 +1156,7 @@ def __init__(self, config): output_size=config.text_config.hidden_size, hidden_act=config.text_config.hidden_act, ) - self.perceiver_resampler = Idefics2PerceiverResampler._from_config( - config, attn_implementation=config._attn_implementation["perceiver_config"] - ) + self.perceiver_resampler = Idefics2PerceiverResampler(config) def forward(self, image_hidden_states, attention_mask): image_hidden_states = self.modality_projection(image_hidden_states) diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index bd64e5db681b..8261389503a4 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -824,17 +824,19 @@ def __init__(self, config: Idefics3Config): self.vocab_size = self.config.text_config.vocab_size self.vision_model = Idefics3VisionTransformer._from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.connector = Idefics3Connector(config) - self.text_model = AutoModel.from_config(config.text_config, attn_implementation=config._attn_implementation) + self.text_model = AutoModel.from_config( + config.text_config, attn_implementation=config._attn_implementation["text_config"] + ) self.image_seq_len = int( ((config.vision_config.image_size // config.vision_config.patch_size) ** 2) / (config.scale_factor**2) ) self.image_token_id = self.config.image_token_id - self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2" + self._use_flash_attention_2 = "flash_attention_2" in config._attn_implementation.values() self.post_init() diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 0c4f056bd441..a89ec5254503 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -315,7 +315,6 @@ class InstructBlipPreTrainedModel(PreTrainedModel): config_class = InstructBlipConfig base_model_prefix = "blip" supports_gradient_checkpointing = True - _is_composite = True _no_split_modules = [ "InstructBlipQFormerEmbeddings", @@ -536,8 +535,6 @@ def forward( class InstructBlipVisionModel(InstructBlipPreTrainedModel): main_input_name = "pixel_values" config_class = InstructBlipVisionConfig - # Ignore copy - _is_composite = False def __init__(self, config: InstructBlipVisionConfig): super().__init__(config) @@ -1087,8 +1084,6 @@ class InstructBlipQFormerModel(InstructBlipPreTrainedModel): instruction as input. """ - _is_composite = False - def __init__(self, config: InstructBlipQFormerConfig): super().__init__(config) self.config = config diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 7cb1d3fe5145..ce536b1e5bc7 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -322,7 +322,6 @@ class InstructBlipVideoPreTrainedModel(PreTrainedModel): config_class = InstructBlipVideoConfig base_model_prefix = "blip" supports_gradient_checkpointing = True - _is_composite = True _no_split_modules = [ "InstructBlipVideoQFormerEmbeddings", @@ -544,9 +543,6 @@ class InstructBlipVideoVisionModel(InstructBlipVideoPreTrainedModel): main_input_name = "pixel_values" config_class = InstructBlipVideoVisionConfig - # Ignore copy - _is_composite = False - def __init__(self, config: InstructBlipVideoVisionConfig): super().__init__(config) self.config = config @@ -1095,8 +1091,6 @@ class InstructBlipVideoQFormerModel(InstructBlipVideoPreTrainedModel): instruction as input. """ - _is_composite = True - def __init__(self, config: InstructBlipVideoQFormerConfig): super().__init__(config) self.config = config diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index 2126a0a45acb..641f3ce9dd87 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -125,7 +125,6 @@ class LlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" - _is_composite = True _supports_cache_class = True def _init_weights(self, module): diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 88e93ce8e881..ff9e86df9fe5 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -234,7 +234,6 @@ class LlavaNextPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaNextVisionAttention"] _skip_keys_device_placement = "past_key_values" - _is_composite = True _supports_cache_class = True def _init_weights(self, module): diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 38395fe3aa15..8d97c95a7131 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -279,7 +279,6 @@ class LlavaNextVideoPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["LlavaNextVideoVisionAttention"] _skip_keys_device_placement = "past_key_values" - _is_composite = True _supports_cache_class = True def _init_weights(self, module): diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index c378ff09f1e4..479bcb833a8e 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -364,7 +364,7 @@ class LlavaOnevisionForConditionalGeneration(LlavaOnevisionPreTrainedModel, Gene def __init__(self, config: LlavaOnevisionConfig): super().__init__(config) self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config) @@ -373,7 +373,7 @@ def __init__(self, config: LlavaOnevisionConfig): self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.post_init() diff --git a/src/transformers/models/mllama/modeling_mllama.py b/src/transformers/models/mllama/modeling_mllama.py index 9c31d9abe5ba..31d2bfe0c8e0 100644 --- a/src/transformers/models/mllama/modeling_mllama.py +++ b/src/transformers/models/mllama/modeling_mllama.py @@ -2032,6 +2032,8 @@ def prepare_inputs_for_generation( MLLAMA_START_DOCSTRING, ) class MllamaForConditionalGeneration(MllamaPreTrainedModel, GenerationMixin): + _is_composite = True + def __init__(self, config: MllamaConfig): super().__init__(config) self.vocab_size = config.text_config.vocab_size @@ -2041,10 +2043,10 @@ def __init__(self, config: MllamaConfig): self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.vision_model = MllamaVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.language_model = MllamaForCausalLM._from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.multi_modal_projector = nn.Linear( config.vision_config.vision_output_dim, diff --git a/src/transformers/models/musicgen/modeling_musicgen.py b/src/transformers/models/musicgen/modeling_musicgen.py index 09a8ef79f269..536e9f0dcf22 100644 --- a/src/transformers/models/musicgen/modeling_musicgen.py +++ b/src/transformers/models/musicgen/modeling_musicgen.py @@ -1670,7 +1670,6 @@ class MusicgenForConditionalGeneration(PreTrainedModel, GenerationMixin): supports_gradient_checkpointing = True _supports_flash_attn_2 = True _supports_sdpa = True - _is_composite = True def __init__( self, diff --git a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py index 29a3986cc865..c6ae54aab251 100644 --- a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py +++ b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py @@ -1596,7 +1596,6 @@ class MusicgenMelodyForConditionalGeneration(PreTrainedModel, GenerationMixin): supports_gradient_checkpointing = True _supports_flash_attn_2 = True _supports_sdpa = True - _is_composite = True def __init__( self, diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 4cea67b6ea61..24cf89edc77b 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -192,7 +192,6 @@ class PaliGemmaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["PaliGemmaMultiModalProjector"] _skip_keys_device_placement = "past_key_values" - _is_composite = True _supports_sdpa = True _supports_cache_class = True _supports_quantized_cache = True diff --git a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py index a5ac1f836385..9170fdb2052e 100644 --- a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py +++ b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py @@ -544,6 +544,7 @@ class Qwen2AudioPreTrainedModel(PreTrainedModel): _no_split_modules = ["Qwen2AudioAttention"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): # important: this ported version of Qwen2Audio isn't meant for training from scratch - only @@ -559,14 +560,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @property - def _supports_sdpa(self): - """ - Retrieve language_model's attribute to check whether the model supports - SDPA or not. - """ - return self.language_model._supports_sdpa - QWEN2AUDIOENCODER_START_DOCSTRING = r""" This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the @@ -859,12 +852,14 @@ def forward(self, audio_features): class Qwen2AudioForConditionalGeneration(Qwen2AudioPreTrainedModel, GenerationMixin): def __init__(self, config: Qwen2AudioConfig): super().__init__(config) - self.audio_tower = AutoModel.from_config(config.audio_config, attn_implementation=config._attn_implementation) + self.audio_tower = AutoModel.from_config( + config.audio_config, attn_implementation=config._attn_implementation["audio_config"] + ) self.multi_modal_projector = Qwen2AudioMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py b/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py index 27615eb789f0..08abd53966a2 100644 --- a/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py @@ -17,6 +17,8 @@ import os from typing import Union +from transformers.models.qwen2.configuration_qwen2 import Qwen2Config + from ...configuration_utils import PretrainedConfig from ...modeling_rope_utils import rope_config_validation from ...utils import logging @@ -202,6 +204,7 @@ def __init__( max_window_layers=80, attention_dropout=0.0, vision_config=None, + text_config=None, rope_scaling=None, **kwargs, ): @@ -210,6 +213,31 @@ def __init__( elif vision_config is None: self.vision_config = Qwen2VLVisionConfig() + self.text_config = ( + Qwen2Config(**text_config) + if text_config is not None + else Qwen2Config( + vocab_size=vocab_size, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + num_key_value_heads=num_key_value_heads, + hidden_act=hidden_act, + max_position_embeddings=max_position_embeddings, + initializer_range=initializer_range, + rms_norm_eps=rms_norm_eps, + use_cache=use_cache, + tie_word_embeddings=tie_word_embeddings, + rope_theta=rope_theta, + use_sliding_window=use_sliding_window, + sliding_window=sliding_window, + max_window_layers=max_window_layers, + attention_dropout=attention_dropout, + rope_scaling=rope_scaling, + ) + ) + self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings self.hidden_size = hidden_size diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 85418a134aa1..208c7ee97686 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -49,6 +49,7 @@ logging, replace_return_docstrings, ) +from ..qwen2.configuration_qwen2 import Qwen2Config from .configuration_qwen2_vl import Qwen2VLConfig, Qwen2VLVisionConfig @@ -1081,10 +1082,18 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch. QWEN2VL_START_DOCSTRING, ) class Qwen2VLModel(Qwen2VLPreTrainedModel): - def __init__(self, config: Qwen2VLConfig): + def __init__(self, config: Qwen2Config): super().__init__(config) self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size + if hasattr(config, "text_config"): + config = config.get_text_config() + config._attn_implementation = ( + config._attn_implementation if config._attn_implementation is not None else "eager" + ) + logger.warning_once( + "If you are loading Qwen2Model directly, make sure to load with `config.text_config` in the ipnut arguments" + ) self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) self.layers = nn.ModuleList( @@ -1424,9 +1433,11 @@ class Qwen2VLForConditionalGeneration(Qwen2VLPreTrainedModel, GenerationMixin): def __init__(self, config): super().__init__(config) self.visual = Qwen2VisionTransformerPretrainedModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] + ) + self.model = Qwen2VLModel._from_config( + config.text_config, attn_implementation=config._attn_implementation["text_config"] ) - self.model = Qwen2VLModel(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) self.padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index 0872f163579d..f2d1d10e3773 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -669,7 +669,6 @@ class SiglipPreTrainedModel(PreTrainedModel): config_class = SiglipConfig base_model_prefix = "siglip" supports_gradient_checkpointing = True - _is_composite = True _no_split_modules = [ "SiglipTextEmbeddings", @@ -1000,7 +999,6 @@ def forward( ) class SiglipTextModel(SiglipPreTrainedModel): config_class = SiglipTextConfig - _is_composite = False def __init__(self, config: SiglipTextConfig): super().__init__(config) @@ -1143,7 +1141,6 @@ def forward(self, hidden_state): class SiglipVisionModel(SiglipPreTrainedModel): config_class = SiglipVisionConfig main_input_name = "pixel_values" - _is_composite = False def __init__(self, config: SiglipVisionConfig): super().__init__(config) @@ -1454,7 +1451,6 @@ def forward( ) class SiglipForImageClassification(SiglipPreTrainedModel): main_input_name = "pixel_values" - _is_composite = False def __init__(self, config: SiglipConfig) -> None: super().__init__(config) @@ -1464,7 +1460,7 @@ def __init__(self, config: SiglipConfig) -> None: # Create the vision model with proper attention # and take only vision_model submodule (for backward compatibility) vision_model = SiglipVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation + config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) self.vision_model = vision_model.vision_model diff --git a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py index 8d1b86908b75..012ab8dfd84d 100644 --- a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py +++ b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py @@ -182,7 +182,6 @@ class SpeechEncoderDecoderModel(PreTrainedModel): main_input_name = "inputs" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False - _is_composite = True def __init__( self, diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 35f55a52d8bd..05e633d56e44 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -126,7 +126,6 @@ class VideoLlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["VideoLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" - _is_composite = True _supports_cache_class = True def _init_weights(self, module): diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index 559964bd6ce3..e7ef915362d1 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -132,7 +132,6 @@ class VipLlavaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["VipLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" - _is_composite = True _supports_cache_class = True def _init_weights(self, module): @@ -277,7 +276,7 @@ def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_m return model_embeds # Ignore copy - def get_image_features(self, pixel_values: torch.FloatTensor, vision_feature_layers: list[int]): + def get_image_features(self, pixel_values: torch.FloatTensor, vision_feature_layers: List[int]): image_outputs = self.vision_tower(pixel_values, output_hidden_states=True) # For VIP-llava, the image features are computed this way diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py index 353c332d3ee9..3908b315e01e 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py @@ -160,7 +160,6 @@ class VisionEncoderDecoderModel(PreTrainedModel): main_input_name = "pixel_values" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False - _is_composite = True def __init__( self, diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 5babc62ef649..653bf8963a0c 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -161,7 +161,6 @@ def clip_loss(similarity: torch.Tensor) -> torch.Tensor: class VisionTextDualEncoderModel(PreTrainedModel): config_class = VisionTextDualEncoderConfig base_model_prefix = "vision_text_dual_encoder" - _is_composite = True def __init__( self, diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index b36059933c4d..43b83f85d78e 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -445,11 +445,7 @@ class BlipModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_pruning = False test_resize_embeddings = False test_attention_outputs = False - - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True def setUp(self): self.model_tester = BlipModelTester(self) @@ -474,6 +470,10 @@ def test_retain_grad_hidden_states_attentions(self): def test_model_get_set_embeddings(self): pass + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_sdpa_can_dispatch_composite_models(self): + pass + # override as the `logit_scale` parameter initilization is different for Blip def test_initialization(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -809,11 +809,7 @@ class BlipVQAModelTest(ModelTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True def setUp(self): self.model_tester = BlipVQAModelTester(self) @@ -884,6 +880,10 @@ def test_inputs_embeds(self): def test_model_get_set_embeddings(self): pass + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_sdpa_can_dispatch_composite_models(self): + pass + @require_torch class BlipTextRetrievalModelTest(ModelTesterMixin, unittest.TestCase): @@ -894,11 +894,7 @@ class BlipTextRetrievalModelTest(ModelTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True def setUp(self): self.model_tester = BlipTextRetrievalModelTester(self) @@ -923,6 +919,10 @@ def test_retain_grad_hidden_states_attentions(self): def test_model_get_set_embeddings(self): pass + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_sdpa_can_dispatch_composite_models(self): + pass + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index a2d2eeebbd99..aff94374d457 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -456,11 +456,7 @@ class Blip2ForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, GenerationT test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - - # We define thsi flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True def setUp(self): self.model_tester = Blip2ForConditionalGenerationDecoderOnlyModelTester(self) @@ -493,6 +489,10 @@ def test_save_load_fast_init_from_base(self): def test_save_load_fast_init_to_base(self): pass + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_sdpa_can_dispatch_composite_models(self): + pass + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() @@ -720,11 +720,7 @@ class Blip2ModelTest(ModelTesterMixin, PipelineTesterMixin, GenerationTesterMixi test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True # TODO: Fix the failed tests def is_pipeline_test_to_skip( @@ -771,6 +767,10 @@ def test_save_load_fast_init_to_base(self): def test_cpu_offload(self): pass + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_sdpa_can_dispatch_composite_models(self): + pass + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/models/clip/test_modeling_clip.py b/tests/models/clip/test_modeling_clip.py index 14ba4259ec49..b8ce4163b8d5 100644 --- a/tests/models/clip/test_modeling_clip.py +++ b/tests/models/clip/test_modeling_clip.py @@ -191,12 +191,64 @@ class CLIPModelTesterMixin(ModelTesterMixin): different output logits, and are not supposed to be used or tested with padding_side="left". """ + def test_sdpa_can_dispatch_composite_models(self): + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + + # Load the model with SDPA + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + # Load model with eager attention + model_eager = model_class.from_pretrained( + tmpdirname, + attn_implementation="eager", + ) + model_eager = model_eager.eval().to(torch_device) + + # SigLip has one shared cls attr for all models, so we assign both submodels heer + vision_attn = text_attn = "sdpa" if model._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + if hasattr(model_sdpa, "vision_model") and hasattr(model_sdpa, "text_model"): + self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.text_model.config._attn_implementation == text_attn) + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") + + if hasattr(model_sdpa.config, "text_config"): + self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} + ) + else: + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_eager.config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + def test_eager_matches_sdpa_inference( self, torch_dtype: str, use_attention_mask_options: Tuple[Optional[str], ...] = (None, "left", "right"), logit_keys: Tuple[str, ...] = ("logits_per_image", "logits_per_text", "image_embeds", "text_embeds"), - is_composite: bool = True, ): if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") @@ -253,39 +305,6 @@ def get_mean_reldiff(msg, current_case, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - if not is_composite: - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - self.assertTrue(model_eager.config._attn_implementation == "eager") - else: - # CLIP has one shared cls attr for all models, so both submodels are SDPA or eager - # We expect `None` as it is the requested one which will be assigned to each sub-config - # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - vision_attn = text_attn = "sdpa" if model._supports_sdpa else "eager" - self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) - self.assertTrue(model_sdpa.text_model.config._attn_implementation == text_attn) - self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) - - self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") - self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} - ) - - for name, submodule in model_eager.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - has_sdpa = True - break - - if not has_sdpa: - raise ValueError("The SDPA model should have SDPA attention layers") - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving the model each time, # but it would be nicer to have an efficient way to use parameterized.expand cases = [ @@ -475,9 +494,12 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("last_hidden_state", "pooler_output", "image_embeds"), use_attention_mask_options=(None,), - is_composite=False, ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + class CLIPTextModelTester: def __init__( @@ -654,9 +676,12 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("last_hidden_state", "pooler_output", "text_embeds"), use_attention_mask_options=(None, "right"), # "left" is not supported for text model - is_composite=False, ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + @require_torch_sdpa def test_sdpa_can_dispatch_on_flash(self): self.skipTest(reason="CLIPTextModel has two attention masks: `causal_attention_mask` and `attention_mask`") @@ -722,6 +747,7 @@ class CLIPModelTest(CLIPModelTesterMixin, PipelineTesterMixin, unittest.TestCase test_pruning = False test_resize_embeddings = False test_attention_outputs = False + _is_composite = True def setUp(self): self.model_tester = CLIPModelTester(self) @@ -993,6 +1019,10 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): use_attention_mask_options=(None, "right"), # "left" is not supported for text model ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + @require_torch_sdpa def test_sdpa_can_dispatch_on_flash(self): self.skipTest(reason="CLIP text tower has two attention masks: `causal_attention_mask` and `attention_mask`") @@ -1122,6 +1152,7 @@ class CLIPForImageClassificationModelTest(CLIPModelTesterMixin, PipelineTesterMi test_pruning = False test_resize_embeddings = False test_attention_outputs = False + _is_composite = True def setUp(self): self.model_tester = CLIPForImageClassificationModelTester(self) @@ -1159,9 +1190,12 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("logits",), use_attention_mask_options=(None,), - is_composite=False, ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/gemma2/test_modeling_gemma2.py b/tests/models/gemma2/test_modeling_gemma2.py index 4e7b3553460f..b8d44fb10da1 100644 --- a/tests/models/gemma2/test_modeling_gemma2.py +++ b/tests/models/gemma2/test_modeling_gemma2.py @@ -86,6 +86,11 @@ def setUp(self): def test_model_outputs_equivalence(self, **kwargs): pass + unittest.skip("Gemma2's forcefully disables sdpa due to softcapping") + + def test_sdpa_can_dispatch_non_composite_models(self): + pass + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @unittest.skip("Gemma2's eager attn/sdpa attn outputs are expected to be different") def test_eager_matches_sdpa_inference(self): diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index d3456bdac083..4a5f67bbce12 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -14,23 +14,19 @@ # limitations under the License. """Testing suite for the PyTorch Idefics model.""" -import tempfile import unittest -from parameterized import parameterized - from transformers import BitsAndBytesConfig, IdeficsConfig, is_torch_available, is_vision_available from transformers.testing_utils import ( TestCasePlus, is_pt_tf_cross_test, require_bitsandbytes, require_torch, - require_torch_sdpa, require_vision, slow, torch_device, ) -from transformers.utils import cached_property, is_torch_bf16_available_on_device, is_torch_fp16_available_on_device +from transformers.utils import cached_property from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask @@ -322,6 +318,7 @@ class IdeficsModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_pruning = False test_headmasking = False test_torchscript = False + _is_composite = True def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) @@ -566,57 +563,9 @@ def test_model_from_pretrained(self): model = IdeficsModel.from_pretrained(model_name) self.assertIsNotNone(model) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @require_torch_sdpa - @slow - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): - self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") - - if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): - self.skipTest( - f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" - ) - - # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. - if torch_dtype == "float16": - torch_dtype = torch.float16 - elif torch_dtype == "bfloat16": - torch_dtype = torch.bfloat16 - elif torch_dtype == "float32": - torch_dtype = torch.float32 - - for model_class in self.all_model_classes: - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - model = model_class(config) - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) - model_sdpa = model_sdpa.eval().to(torch_device) - - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - - # Also test that nothing break if we request SDPA explicitly - model_sdpa_explicit = model_class.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) - model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - - self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") - - model_eager = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch_dtype, - attn_implementation="eager", - ) - model_eager = model_eager.eval().to(torch_device) - - self.assertTrue(model_eager.config._attn_implementation == "eager") - - for name, submodule in model_eager.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - raise ValueError("The eager model should not have SDPA attention layers") + @unittest.skip("Idefics has a hard requirement on SDPA") + def test_sdpa_can_dispatch_composite_models(self): + pass @unittest.skipIf(not is_torch_greater_or_equal_than_2_0, reason="pytorch 2.0 or higher is required") @@ -655,6 +604,10 @@ def test_training_gradient_checkpointing_use_reentrant(self): def test_training_gradient_checkpointing_use_reentrant_false(self): pass + @unittest.skip("Idefics has a hard requirement on SDPA") + def test_sdpa_can_dispatch_composite_models(self): + pass + @unittest.skipIf(not is_torch_greater_or_equal_than_2_0, reason="pytorch 2.0 or higher is required") @require_torch diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 3cfef5baa02a..c2f5e712e7c3 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -21,7 +21,6 @@ from io import BytesIO import requests -from parameterized import parameterized from transformers import ( AutoProcessor, @@ -41,7 +40,6 @@ slow, torch_device, ) -from transformers.utils import is_torch_bf16_available_on_device, is_torch_fp16_available_on_device from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester @@ -183,6 +181,7 @@ class Idefics2ModelTest(ModelTesterMixin, unittest.TestCase): test_pruning = False test_resize_embeddings = True test_head_masking = False + _is_composite = True def setUp(self): self.model_tester = Idefics2VisionText2TextModelTester(self) @@ -330,64 +329,31 @@ def test_resize_embeddings_untied(self): # Check that the model can still do a forward pass successfully (every parameter should be resized) model(**self._prepare_for_class(inputs_dict, model_class)) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): - self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") - - if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): - self.skipTest( - f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" - ) - - # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. - if torch_dtype == "float16": - torch_dtype = torch.float16 - elif torch_dtype == "bfloat16": - torch_dtype = torch.bfloat16 - elif torch_dtype == "float32": - torch_dtype = torch.float32 - + def test_sdpa_can_dispatch_composite_models(self): for model_class in self.all_model_classes: config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() model = model_class(config) + with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) - model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) - # see https://github.com/huggingface/transformers/pull/32238 - # we put `None` because that is the requested attn implementation, which will dispatch to SDPA internally if available - perceiver_attn = None if model.connector.perceiver_resampler._supports_sdpa else "eager" vision_attn = None if model.vision_model._supports_sdpa else "eager" self.assertTrue( model_sdpa.config._attn_implementation == {"text_config": None, "perceiver_config": None, "vision_config": None} ) self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) - self.assertTrue(model_sdpa.connector.perceiver_resampler.config._attn_implementation == perceiver_attn) - - # Also test that nothing break if we request SDPA explicitly - # If the model supports sdpa (i.e. one of sub-models supports it) we'll raise error because we - # explicitly asked for SDPA, in comparison to above when SDPA dispatches by default is it is available. - with self.assertRaises(ValueError): - _ = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa") - - model_eager = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch_dtype, - attn_implementation="eager", - ) - model_eager = model_eager.eval().to(torch_device) + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) self.assertTrue( model_eager.config._attn_implementation == {"text_config": "eager", "perceiver_config": "eager", "vision_config": "eager"} ) self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") - self.assertTrue(model_eager.connector.perceiver_resampler.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ @@ -400,7 +366,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: has_sdpa = True break - if not has_sdpa: + if not has_sdpa and model_sdpa.config.model_type != "falcon": raise ValueError("The SDPA model should have SDPA attention layers") diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 2c836c069b7b..37c5fac8aec4 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -460,11 +460,7 @@ class InstructBlipForConditionalGenerationDecoderOnlyTest(ModelTesterMixin, Gene test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True def setUp(self): self.model_tester = InstructBlipForConditionalGenerationDecoderOnlyModelTester(self) @@ -534,6 +530,10 @@ def test_model_from_pretrained(self): model = InstructBlipForConditionalGeneration.from_pretrained(model_name) self.assertIsNotNone(model) + @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") + def test_sdpa_can_dispatch_composite_models(self): + pass + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index eb70f6ff2375..589bb5e7629c 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -481,11 +481,7 @@ class InstructBlipVideoForConditionalGenerationDecoderOnlyTest( test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True def setUp(self): self.model_tester = InstructBlipVideoForConditionalGenerationDecoderOnlyModelTester(self) @@ -555,6 +551,10 @@ def test_model_from_pretrained(self): model = InstructBlipVideoForConditionalGeneration.from_pretrained(model_name) self.assertIsNotNone(model) + @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") + def test_sdpa_can_dispatch_composite_models(self): + pass + # We will verify our results on an image of cute cats def prepare_video(): diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index 431aab1f851c..00fafa29e8cc 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -257,11 +257,7 @@ class Kosmos2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_pruning = False test_resize_embeddings = False test_attention_outputs = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because LM/vision models used in tests don't support SDPA - supports_sdpa = False + _is_composite = True # TODO: `image-to-text` pipeline for this model needs Processor. def is_pipeline_test_to_skip( @@ -512,6 +508,10 @@ def _create_and_check_torchscript(self, config, inputs_dict): # (Even with this call, there are still memory leak by ~0.04MB) self.clear_torch_jit_class_registry() + @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") + def test_sdpa_can_dispatch_composite_models(self): + pass + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/llava/test_modeling_llava.py b/tests/models/llava/test_modeling_llava.py index f66c3394238b..28816f040c8a 100644 --- a/tests/models/llava/test_modeling_llava.py +++ b/tests/models/llava/test_modeling_llava.py @@ -186,11 +186,7 @@ class LlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterM pipeline_model_mapping = {"image-to-text": LlavaForConditionalGeneration} if is_torch_available() else {} test_pruning = False test_head_masking = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA - supports_sdpa = True + _is_composite = True def setUp(self): self.model_tester = LlavaVisionText2TextModelTester(self) diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index 57633a7fd17a..eaa6e6ae53ba 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -217,11 +217,7 @@ class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, unittest.Test all_generative_model_classes = (LlavaNextForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA - supports_sdpa = True + _is_composite = True def setUp(self): self.model_tester = LlavaNextVisionText2TextModelTester(self) diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index a53b8a45f84d..242e35b416e2 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -235,11 +235,7 @@ class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, unittest all_generative_model_classes = (LlavaNextVideoForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA - supports_sdpa = True + _is_composite = True def setUp(self): self.model_tester = LlavaNextVideoVisionText2TextModelTester(self) diff --git a/tests/models/llava_onevision/test_modeling_llava_onevision.py b/tests/models/llava_onevision/test_modeling_llava_onevision.py index 0e9c88cb3463..fb890e072ea1 100644 --- a/tests/models/llava_onevision/test_modeling_llava_onevision.py +++ b/tests/models/llava_onevision/test_modeling_llava_onevision.py @@ -219,6 +219,7 @@ class LlavaOnevisionForConditionalGenerationModelTest(ModelTesterMixin, Generati all_generative_model_classes = (LlavaOnevisionForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False + _is_composite = True def setUp(self): self.model_tester = LlavaOnevisionVisionText2TextModelTester(self) diff --git a/tests/models/mllama/test_modeling_mllama.py b/tests/models/mllama/test_modeling_mllama.py index f31957d78aa8..d7d869e905fe 100644 --- a/tests/models/mllama/test_modeling_mllama.py +++ b/tests/models/mllama/test_modeling_mllama.py @@ -281,6 +281,7 @@ class MllamaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTester test_pruning = False test_head_masking = False test_torchscript = False + _is_composite = True def setUp(self): self.model_tester = MllamaVisionText2TextModelTester(self) diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 9a1cf25b19c6..0aeedc9b7b65 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -690,8 +690,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - model_eager = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, @@ -699,20 +697,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") - - for name, submodule in model_eager.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 8 times the model, # but it would be nicer to have an efficient way to use parameterized.expand fail_cases = [] @@ -1075,6 +1059,7 @@ class MusicgenTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, # not to test torchscript as the model tester doesn't prepare `input_values` and `padding_mask` # (and `torchscript` hates `None` values). test_torchscript = False + _is_composite = True def setUp(self): self.model_tester = MusicgenTester(self) @@ -1933,6 +1918,63 @@ def test_flash_attn_2_generate_use_cache(self): use_cache=True, ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" + text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) + self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) + self.assertTrue( + model_sdpa.config._attn_implementation + == { + "audio_encoder": None, + "text_encoder": None, + "decoder": None, + } + ) + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation + == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} + ) + + for name, submodule in model_eager.named_modules(): + if "SdpaAttention" in submodule.__class__.__name__: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + if "SdpaAttention" in submodule.__class__.__name__: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow @@ -1998,24 +2040,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" - text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" - decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - - # `None` as it is the requested one which will be assigned to each sub-config - # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) - self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) - self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) - self.assertTrue( - model_sdpa.config._attn_implementation - == { - "audio_encoder": None, - "text_encoder": None, - "decoder": None, - } - ) - model_eager = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, @@ -2023,26 +2047,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") - self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") - self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") - self.assertTrue( - model_eager.config._attn_implementation - == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} - ) - - for name, submodule in model_eager.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 8 times the model, # but it would be nicer to have an efficient way to use parameterized.expand fail_cases = [] diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index cbd361f09628..fe464fc8d094 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -688,8 +688,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - model_eager = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, @@ -697,20 +695,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == "eager") - - for name, submodule in model_eager.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 8 times the model, # but it would be nicer to have an efficient way to use parameterized.expand fail_cases = [] @@ -1075,6 +1059,7 @@ class MusicgenMelodyTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester # not to test torchscript as the model tester doesn't prepare `input_features` and `padding_mask` # (and `torchscript` hates `None` values). test_torchscript = False + _is_composite = True def setUp(self): self.model_tester = MusicgenMelodyTester(self) @@ -1251,6 +1236,63 @@ def test_gradient_checkpointing_backward_compatibility(self): model = model_class(config) self.assertTrue(model.is_gradient_checkpointing) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" + text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) + self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) + self.assertTrue( + model_sdpa.config._attn_implementation + == { + "audio_encoder": None, + "text_encoder": None, + "decoder": None, + } + ) + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation + == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} + ) + + for name, submodule in model_eager.named_modules(): + if "SdpaAttention" in submodule.__class__.__name__: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + if "SdpaAttention" in submodule.__class__.__name__: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + @unittest.skip(reason="MusicGen has multiple inputs embeds and lm heads that should not be tied.") def test_tie_model_weights(self): pass @@ -1975,24 +2017,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" - text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" - decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - - # `None` as it is the requested one which will be assigned to each sub-config - # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) - self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) - self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) - self.assertTrue( - model_sdpa.config._attn_implementation - == { - "audio_encoder": None, - "text_encoder": None, - "decoder": None, - } - ) - model_eager = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, @@ -2000,26 +2024,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") - self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") - self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") - self.assertTrue( - model_eager.config._attn_implementation - == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} - ) - - for name, submodule in model_eager.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 8 times the model, # but it would be nicer to have an efficient way to use parameterized.expand fail_cases = [] diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index 2b8535efa69c..1476cb5fd3d5 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -186,11 +186,7 @@ class PaliGemmaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTes test_pruning = False test_torchscript = False test_head_masking = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA - supports_sdpa = True + _is_composite = True def setUp(self): self.model_tester = PaliGemmaVisionText2TextModelTester(self) diff --git a/tests/models/qwen2_audio/test_modeling_qwen2_audio.py b/tests/models/qwen2_audio/test_modeling_qwen2_audio.py index 4054055082c7..819c3ed46961 100644 --- a/tests/models/qwen2_audio/test_modeling_qwen2_audio.py +++ b/tests/models/qwen2_audio/test_modeling_qwen2_audio.py @@ -15,6 +15,7 @@ """Testing suite for the PyTorch Qwen2Audio model.""" import gc +import tempfile import unittest from io import BytesIO from urllib.request import urlopen @@ -29,6 +30,7 @@ ) from transformers.testing_utils import ( require_torch, + require_torch_sdpa, slow, torch_device, ) @@ -152,6 +154,7 @@ class Qwen2AudioForConditionalGenerationModelTest(ModelTesterMixin, unittest.Tes all_model_classes = (Qwen2AudioForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False + _is_composite = True def setUp(self): self.model_tester = Qwen2AudioModelTester(self) @@ -165,6 +168,57 @@ def test_sdpa_can_compile_dynamic(self): def test_sdpa_can_dispatch_on_flash(self): pass + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + # overwrite because Qwen2 is audio+text model (not vision+text) + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + audio_model_sdpa = getattr(model, "audio_tower") + language_model_sdpa = getattr(model, "language_model") + text_attn = "sdpa" if language_model_sdpa._supports_sdpa else "eager" + vision_attn = "sdpa" if audio_model_sdpa._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "audio_config": None}) + self.assertTrue(language_model_sdpa.config._attn_implementation == text_attn) + self.assertTrue(audio_model_sdpa.config._attn_implementation == vision_attn) + + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "audio_config": "eager"} + ) + self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "audio_tower").config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + @require_torch class Qwen2AudioForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py index 956243dccebe..9d0be00ea7bd 100644 --- a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py +++ b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py @@ -226,6 +226,7 @@ class Qwen2VLModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCas all_generative_model_classes = (Qwen2VLForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False + _is_composite = True def setUp(self): self.model_tester = Qwen2VLVisionText2TextModelTester(self) diff --git a/tests/models/siglip/test_modeling_siglip.py b/tests/models/siglip/test_modeling_siglip.py index 47ba43e49142..40704b740d76 100644 --- a/tests/models/siglip/test_modeling_siglip.py +++ b/tests/models/siglip/test_modeling_siglip.py @@ -71,12 +71,62 @@ class SiglipModelTesterMixin(ModelTesterMixin): + def test_sdpa_can_dispatch_composite_models(self): + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + + # Load the model with SDPA + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + # Load model with eager attention + model_eager = model_class.from_pretrained( + tmpdirname, + attn_implementation="eager", + ) + model_eager = model_eager.eval().to(torch_device) + + # SigLip has one shared cls attr for all models, so we assign both submodels heer + vision_attn = text_attn = "sdpa" if model._supports_sdpa else "eager" + + if hasattr(model_sdpa, "vision_model") and hasattr(model_sdpa, "text_model"): + self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.text_model.config._attn_implementation == text_attn) + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") + + if hasattr(model_sdpa.config, "text_config"): + self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} + ) + else: + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_eager.config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + def test_eager_matches_sdpa_inference( self, torch_dtype: str, use_attention_mask_options: Tuple[bool, ...] = (True, False), logit_keys: Tuple[str, ...] = ("logits_per_image", "logits_per_text", "image_embeds", "text_embeds"), - is_composite: bool = True, ): if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") @@ -133,39 +183,6 @@ def get_mean_reldiff(msg, current_case, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - if not is_composite: - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - self.assertTrue(model_eager.config._attn_implementation == "eager") - else: - # SigLip has one shared cls attr for all models, so we assign both submodels heer - vision_attn = text_attn = "sdpa" if model._supports_sdpa else "eager" - - # `None` as it is the requested one which will be assigned to each sub-config - # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) - self.assertTrue(model_sdpa.text_model.config._attn_implementation == text_attn) - self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) - - self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") - self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} - ) - - for name, submodule in model_eager.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving the model each time, # but it would be nicer to have an efficient way to use parameterized.expand cases = [ @@ -415,9 +432,12 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("pooler_output", "last_hidden_state"), use_attention_mask_options=(False,), - is_composite=False, ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + class SiglipTextModelTester: def __init__( @@ -578,9 +598,12 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): torch_dtype=torch_dtype, logit_keys=("pooler_output", "last_hidden_state"), use_attention_mask_options=(False, True), - is_composite=False, ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + class SiglipModelTester: def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=True): @@ -648,6 +671,7 @@ class SiglipModelTest(SiglipModelTesterMixin, PipelineTesterMixin, unittest.Test test_cpu_offload = False test_disk_offload_safetensors = False test_disk_offload_bin = False + _is_composite = True # Copied from tests.models.clip.test_modeling_clip.CLIPModelTest.setUp with CLIP->Siglip def setUp(self): @@ -870,6 +894,10 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): use_attention_mask_options=(False, True), ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + class SiglipForImageClassificationModelTester(SiglipModelTester): def __init__(self, parent): @@ -907,6 +935,7 @@ class SiglipForImageClassificationModelTest(SiglipModelTesterMixin, PipelineTest test_cpu_offload = False test_disk_offload_safetensors = False test_disk_offload_bin = False + _is_composite = True def setUp(self): self.model_tester = SiglipForImageClassificationModelTester(self) @@ -941,9 +970,13 @@ def test_initialization(self): @is_flaky() def test_eager_matches_sdpa_inference(self, torch_dtype: str): super().test_eager_matches_sdpa_inference( - torch_dtype=torch_dtype, logit_keys=("logits",), use_attention_mask_options=(False,), is_composite=False + torch_dtype=torch_dtype, logit_keys=("logits",), use_attention_mask_options=(False,) ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + super().test_sdpa_can_dispatch_composite_models() + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index dbe500c04543..963716ef32c8 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -205,11 +205,7 @@ class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.Tes test_pruning = False test_resize_embeddings = True test_head_masking = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA - supports_sdpa = True + _is_composite = True def setUp(self): self.model_tester = VideoLlavaVisionText2TextModelTester(self) diff --git a/tests/models/vipllava/test_modeling_vipllava.py b/tests/models/vipllava/test_modeling_vipllava.py index 1c93b8af107c..9463ac00848d 100644 --- a/tests/models/vipllava/test_modeling_vipllava.py +++ b/tests/models/vipllava/test_modeling_vipllava.py @@ -168,11 +168,7 @@ class VipLlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTest test_pruning = False test_resize_embeddings = True test_head_masking = False - - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to True because LM/vision models used in tests support SDPA - supports_sdpa = True + _is_composite = True def setUp(self): self.model_tester = VipLlavaVisionText2TextModelTester(self) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 025299e2a62a..baa49a427fef 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -187,6 +187,7 @@ class ModelTesterMixin: test_model_parallel = False is_encoder_decoder = False has_attentions = True + _is_composite = False model_split_percents = [0.5, 0.7, 0.9] def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): @@ -3775,6 +3776,133 @@ def test_flash_attn_2_generate_padding_right(self): self.assertTrue(torch.allclose(out, out_fa)) + def test_attn_implementation_composite_models(self): + """ + Tests if composite models can receive a dict object as attn_implementation, where each key should be + one of the sub-configs from the model's config. + """ + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + for model_class in self.all_model_classes: + if not self._is_composite: + self.skipTest("Model is not a composite model.") + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + sub_configs = { + key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) + } + + # set eager as it will be the one supported in all models + # we just need to test if passing a dict 'attn_implementation' fails or not + attn_implementation_per_subconfig = {} + for key, sub_config in sub_configs.items(): + attn_implementation_per_subconfig[key] = "eager" + + config._attn_implementation = attn_implementation_per_subconfig + model = model_class(config) + self.assertTrue(model.config._attn_implementation == attn_implementation_per_subconfig) + for name, submodule in model.named_modules(): + class_name = submodule.__class__.__name__ + if ( + "SdpaAttention" in class_name + or "SdpaSelfAttention" in class_name + or "FlashAttention" in class_name + ): + raise ValueError("The eager model should not have SDPA/FA2 attention layers") + + @require_torch_sdpa + def test_sdpa_can_dispatch_non_composite_models(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self.all_model_classes[0]._supports_sdpa or self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + self.assertTrue(model_eager.config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + vision_model_names = {"visual", "image_tower", "vision_tower", "vision_model"} + language_model_names = {"language_model", "model", "text_model"} + vision_model_name = [name for name in vision_model_names if hasattr(model_sdpa, name)][0] + language_model_name = [name for name in language_model_names if hasattr(model_sdpa, name)][0] + + vision_model_sdpa = getattr(model, vision_model_name) + language_model_sdpa = getattr(model, language_model_name) + text_attn = "sdpa" if language_model_sdpa._supports_sdpa else "eager" + vision_attn = "sdpa" if vision_model_sdpa._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) + self.assertTrue(language_model_sdpa.config._attn_implementation == text_attn) + self.assertTrue(vision_model_sdpa.config._attn_implementation == vision_attn) + + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + self.assertTrue( + model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} + ) + self.assertTrue(getattr(model_eager, language_model_name).config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, vision_model_name).config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow @@ -3782,9 +3910,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if (not self.all_model_classes[0]._supports_sdpa and not self.all_model_classes[0]._is_composite) or ( - self.all_model_classes[0]._is_composite and not self.supports_sdpa - ): + if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -3839,7 +3965,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): # This means that the class needs to be instantiated much later, after `use_mask` is set, which means a significant refactor of the code. # However masking there is not done at any layers that matters (i.e self-attention), therefore we can safely deactivate it. deactivate_mask = "use_mask_token" in inspect.signature(model_class).parameters - is_encoder_decoder = model.config.is_encoder_decoder with tempfile.TemporaryDirectory() as tmpdirname: @@ -3847,21 +3972,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): model_sdpa = model_class.from_pretrained(tmpdirname, torch_dtype=torch_dtype) model_sdpa = model_sdpa.eval().to(torch_device) - if model_sdpa._is_composite: - vision_model_name = "image_tower" if hasattr(model_sdpa, "image_tower") else "vision_tower" - vision_attn = "sdpa" if getattr(model, vision_model_name)._supports_sdpa else "eager" - text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" - - # `None` as it is the requested one which will be assigned to each sub-config - # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue( - model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None} - ) - self.assertTrue(model_sdpa.language_model.config._attn_implementation == text_attn) - self.assertTrue(getattr(model_sdpa, vision_model_name).config._attn_implementation == vision_attn) - else: - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - model_eager = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, @@ -3869,29 +3979,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): ) model_eager = model_eager.eval().to(torch_device) - if model_eager._is_composite: - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} - ) - self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, vision_model_name).config._attn_implementation == "eager") - else: - self.assertTrue(model_eager.config._attn_implementation == "eager") - - for name, submodule in model_eager.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - # We use these for loops instead of parameterized.expand just for the interest of avoiding loading/saving 16 times the model, # but it would be nicer to have an efficient way to use parameterized.expand fail_cases = [] @@ -4113,9 +4200,7 @@ def test_sdpa_can_dispatch_on_flash(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if (not model_class._is_composite and not model_class._supports_sdpa) or ( - model_class._is_composite and not self.supports_sdpa - ): + if not self._is_composite and not model_class._supports_sdpa: self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4163,9 +4248,7 @@ def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if (not model_class._is_composite and not model_class._supports_sdpa) or ( - model_class._is_composite and not self.supports_sdpa - ): + if not self._is_composite and not model_class._supports_sdpa: self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4207,9 +4290,7 @@ def test_eager_matches_sdpa_generate(self): self.skipTest(f"{self.__class__.__name__} tests a model that does support generate: skipping this test") for model_class in self.all_generative_model_classes: - if (not model_class._is_composite and not model_class._supports_sdpa) or ( - model_class._is_composite and not self.supports_sdpa - ): + if not self._is_composite and not model_class._supports_sdpa: self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4235,21 +4316,6 @@ def test_eager_matches_sdpa_generate(self): low_cpu_mem_usage=True, ).to(torch_device) - if model_sdpa._is_composite: - vision_model_name = "image_tower" if hasattr(model_sdpa, "image_tower") else "vision_tower" - vision_attn = "sdpa" if getattr(model, vision_model_name)._supports_sdpa else "eager" - text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" - - # `None` as it is the requested one which will be assigned to each sub-config - # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue( - model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None} - ) - self.assertTrue(model_sdpa.language_model.config._attn_implementation == text_attn) - self.assertTrue(getattr(model_sdpa, vision_model_name).config._attn_implementation == vision_attn) - else: - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - model_eager = model_class.from_pretrained( tmpdirname, torch_dtype=torch.float16, @@ -4257,29 +4323,6 @@ def test_eager_matches_sdpa_generate(self): attn_implementation="eager", ).to(torch_device) - if model_eager._is_composite: - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} - ) - self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, vision_model_name).config._attn_implementation == "eager") - else: - self.assertTrue(model_eager.config._attn_implementation == "eager") - - for name, submodule in model_eager.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - class_name = submodule.__class__.__name__ - if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: - has_sdpa = True - break - if not has_sdpa: - raise ValueError("The SDPA model should have SDPA attention layers") - # Just test that a large cache works as expected res_eager = model_eager.generate( dummy_input, attention_mask=dummy_attention_mask, max_new_tokens=max_new_tokens, do_sample=False @@ -4427,7 +4470,7 @@ def test_flash_attn_2_can_dispatch_composite_models(self): for model_class in self.all_model_classes: config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() model = model_class(config) - if not model_class._is_composite: + if not self._is_composite: self.skipTest("This model is not a composte model!") with tempfile.TemporaryDirectory() as tmpdirname: @@ -4740,41 +4783,6 @@ def test_flash_attn_2_from_config(self): self.assertFalse(fa2_correctly_converted) - def test_attn_implementation_composite_models(self): - """ - Tests if composite models can receive a dict object as attn_implementation, where each key should be - one of the sub-configs from the model's config. - """ - if not self.has_attentions: - self.skipTest(reason="Model architecture does not support attentions") - - for model_class in self.all_model_classes: - if not model_class._is_composite: - self.skipTest("Model is not a composite model.") - - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - sub_configs = { - key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) - } - - # set eager as it will be the one supported in all models, for the sake of testing - # if passing a dicr fails or not - attn_implementation_per_subconfig = {} - for key, sub_config in sub_configs.items(): - attn_implementation_per_subconfig[key] = "eager" - - config._attn_implementation = attn_implementation_per_subconfig - model = model_class(config) - self.assertTrue(model.config._attn_implementation == attn_implementation_per_subconfig) - for name, submodule in model.named_modules(): - class_name = submodule.__class__.__name__ - if ( - "SdpaAttention" in class_name - or "SdpaSelfAttention" in class_name - or "FlashAttention" in class_name - ): - raise ValueError("The eager model should not have SDPA/FA2 attention layers") - def _get_custom_4d_mask_test_data(self): # Sequence in which all but the last token is the same input_ids = torch.tensor( From 04aba9fd53e0e8b27480f85ff4c28ff442ae7ff7 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 3 Oct 2024 17:41:27 +0200 Subject: [PATCH 53/68] fix copies --- .../models/idefics3/modeling_idefics3.py | 2 +- .../modeling_llava_next_video.py | 2 +- .../models/qwen2_vl/configuration_qwen2_vl.py | 4 +- .../test_modeling_musicgen_melody.py | 114 +++++++++--------- 4 files changed, 62 insertions(+), 60 deletions(-) diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 8261389503a4..196fe30144d5 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -625,7 +625,7 @@ class Idefics3PreTrainedModel(PreTrainedModel): # Copied from transformers.models.idefics2.modeling_idefics2.Idefics2PreTrainedModel._init_weights def _init_weights(self, module): std = ( - self.config.initializer_range + self.config.text_config.initializer_range if hasattr(self.config, "initializer_range") else self.config.text_config.initializer_range ) diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 8d97c95a7131..e262609c3c75 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -395,8 +395,8 @@ def __init__( self.vision_tower = AutoModel.from_config( config.vision_config, attn_implementation=config._attn_implementation["vision_config"] ) - self.multi_modal_projector = LlavaNextVideoMultiModalProjector(config) + self.multi_modal_projector = LlavaNextVideoMultiModalProjector(config) embed_std = 1 / math.sqrt(config.text_config.hidden_size) self.image_newline = nn.Parameter(torch.randn(config.text_config.hidden_size, dtype=self.dtype) * embed_std) diff --git a/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py b/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py index 08abd53966a2..4ec538873c89 100644 --- a/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py @@ -130,6 +130,8 @@ class Qwen2VLConfig(PretrainedConfig): The dropout ratio for the attention probabilities. vision_config (`Dict`, *optional*): The config for the visual encoder initialization. + text_config (`Dict`, *optional*): + The config for the language model initialization. rope_scaling (`Dict`, *optional*): Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value @@ -215,7 +217,7 @@ def __init__( self.text_config = ( Qwen2Config(**text_config) - if text_config is not None + if isinstance(text_config, dict) else Qwen2Config( vocab_size=vocab_size, hidden_size=hidden_size, diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index fe464fc8d094..7398c58ab4d6 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -1236,63 +1236,6 @@ def test_gradient_checkpointing_backward_compatibility(self): model = model_class(config) self.assertTrue(model.is_gradient_checkpointing) - @require_torch_sdpa - def test_sdpa_can_dispatch_composite_models(self): - if not self.has_attentions: - self.skipTest(reason="Model architecture does not support attentions") - - if not self._is_composite: - self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") - - for model_class in self.all_model_classes: - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - model = model_class(config) - - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - model_sdpa = model_class.from_pretrained(tmpdirname) - model_sdpa = model_sdpa.eval().to(torch_device) - - audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" - text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" - decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - - # `None` as it is the requested one which will be assigned to each sub-config - # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) - self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) - self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) - self.assertTrue( - model_sdpa.config._attn_implementation - == { - "audio_encoder": None, - "text_encoder": None, - "decoder": None, - } - ) - model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") - model_eager = model_eager.eval().to(torch_device) - - self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") - self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") - self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") - self.assertTrue( - model_eager.config._attn_implementation - == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} - ) - - for name, submodule in model_eager.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - has_sdpa = True - break - if not has_sdpa and model_sdpa.config.model_type != "falcon": - raise ValueError("The SDPA model should have SDPA attention layers") - @unittest.skip(reason="MusicGen has multiple inputs embeds and lm heads that should not be tied.") def test_tie_model_weights(self): pass @@ -1955,6 +1898,63 @@ def test_flash_attn_2_generate_use_cache(self): use_cache=True, ) + @require_torch_sdpa + def test_sdpa_can_dispatch_composite_models(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + audio_encoder_attn = "sdpa" if model.audio_encoder._supports_sdpa else "eager" + text_encoder_attn = "sdpa" if model.text_encoder._supports_sdpa else "eager" + decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) + self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) + self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) + self.assertTrue( + model_sdpa.config._attn_implementation + == { + "audio_encoder": None, + "text_encoder": None, + "decoder": None, + } + ) + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + + self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") + self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") + self.assertTrue( + model_eager.config._attn_implementation + == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} + ) + + for name, submodule in model_eager.named_modules(): + if "SdpaAttention" in submodule.__class__.__name__: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + if "SdpaAttention" in submodule.__class__.__name__: + has_sdpa = True + break + if not has_sdpa and model_sdpa.config.model_type != "falcon": + raise ValueError("The SDPA model should have SDPA attention layers") + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow From 598b6f5a841186c46e5337949444c2f64ac7ad03 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 3 Oct 2024 20:04:33 +0200 Subject: [PATCH 54/68] let it be this way for now, come back tomorrow to review --- src/transformers/modeling_utils.py | 23 +++++++------- .../grounding_dino/modeling_grounding_dino.py | 2 +- .../omdet_turbo/modeling_omdet_turbo.py | 4 ++- .../models/qwen2_vl/configuration_qwen2_vl.py | 30 ------------------- .../models/qwen2_vl/modeling_qwen2_vl.py | 14 ++------- tests/models/blip/test_modeling_blip.py | 15 ---------- tests/models/idefics/test_modeling_idefics.py | 4 +++ tests/models/kosmos2/test_modeling_kosmos2.py | 4 +++ .../models/qwen2_vl/test_modeling_qwen2_vl.py | 1 - tests/test_modeling_common.py | 1 - 10 files changed, 27 insertions(+), 71 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index c25515aa6456..f79b921b758b 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1581,17 +1581,15 @@ def _autoset_attn_implementation( requested_attn_implementation = config._attn_implementation_internal # Composite models consisting of several PretrainedModels have to specify attention impl as a dict - # where keys are sub-config names. But most people will specify one `str` which means that should dispatch - # for all sub-models or do not specify anything (`None`). - # Below we check is a models is composite and manually prepare a dict of attn impl if not already passed as a dict. - # Later each sub-model will dispatch with its own attn impl, by calling `_from_config(attn_impl="sdpa/FA2/eager")` - # If any of sub-models don't support requested attn, an error will be raised + # where keys are sub-config names. But most people will specify one `str` which means that should dispatch it + # for all sub-models. + # Below we check if a config is composite and manually prepare a dict of attn impl if not already passed as a dict. + # Later each sub-module will dispatch with its own attn impl, by calling `_from_config(attn_impl="sdpa/FA2/eager")` + # If any of sub-modules doesm't support requested attn, an error will be raised. See https://github.com/huggingface/transformers/pull/32238 sub_configs = { key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) } - if sub_configs and all( - name not in cls.__name__.lower() for name in ["chameleon", "dbrx"] - ): # so we have a composite model + if sub_configs and all(name not in cls.__name__.lower() for name in ["dbrx"]): attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): attn_implementation_per_subconfig[key] = ( @@ -1600,8 +1598,13 @@ def _autoset_attn_implementation( else requested_attn_implementation.get(key) ) - config._attn_implementation = attn_implementation_per_subconfig - requested_attn_implementation = config._attn_implementation + # Some models have nested configs where text config holds vision config + # inside itself. So we don't set their attn implementation as dicts and leave + # everything as it was. There are only 3 models like that (Qwen2_VL, GIT, Chameleon) + # and all of them support all attn implementations + if len(attn_implementation_per_subconfig.keys()) != 1: + config._attn_implementation = attn_implementation_per_subconfig + requested_attn_implementation = config._attn_implementation if use_flash_attention_2: logger.warning_once( diff --git a/src/transformers/models/grounding_dino/modeling_grounding_dino.py b/src/transformers/models/grounding_dino/modeling_grounding_dino.py index aaac7488f430..3ba3847d3072 100644 --- a/src/transformers/models/grounding_dino/modeling_grounding_dino.py +++ b/src/transformers/models/grounding_dino/modeling_grounding_dino.py @@ -2118,7 +2118,7 @@ def __init__(self, config: GroundingDinoConfig): # Create text backbone self.text_backbone = AutoModel.from_config( - config.text_config, add_pooling_layer=False, attn_implementation=config._attn_implementation + config.text_config, add_pooling_layer=False, attn_implementation=config._attn_implementation["text_config"] ) self.text_projection = nn.Linear(config.text_config.hidden_size, config.d_model) diff --git a/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py b/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py index bf9dbd951b5b..a8859eb69acf 100644 --- a/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py +++ b/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py @@ -288,7 +288,9 @@ def put(self, key, value) -> None: class OmDetTurboLanguageBackbone(nn.Module): def __init__(self, config: OmDetTurboConfig): super().__init__() - self.model = AutoModel.from_config(config.text_config, attn_implementation=config._attn_implementation) + self.model = AutoModel.from_config( + config.text_config, attn_implementation=config._attn_implementation["text_config"] + ) self.text_projection = nn.Parameter(torch.zeros(config.text_projection_in_dim, config.text_projection_out_dim)) def forward(self, hidden_states, mask=None, encode_type="task"): diff --git a/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py b/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py index 4ec538873c89..27615eb789f0 100644 --- a/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py @@ -17,8 +17,6 @@ import os from typing import Union -from transformers.models.qwen2.configuration_qwen2 import Qwen2Config - from ...configuration_utils import PretrainedConfig from ...modeling_rope_utils import rope_config_validation from ...utils import logging @@ -130,8 +128,6 @@ class Qwen2VLConfig(PretrainedConfig): The dropout ratio for the attention probabilities. vision_config (`Dict`, *optional*): The config for the visual encoder initialization. - text_config (`Dict`, *optional*): - The config for the language model initialization. rope_scaling (`Dict`, *optional*): Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value @@ -206,7 +202,6 @@ def __init__( max_window_layers=80, attention_dropout=0.0, vision_config=None, - text_config=None, rope_scaling=None, **kwargs, ): @@ -215,31 +210,6 @@ def __init__( elif vision_config is None: self.vision_config = Qwen2VLVisionConfig() - self.text_config = ( - Qwen2Config(**text_config) - if isinstance(text_config, dict) - else Qwen2Config( - vocab_size=vocab_size, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - num_hidden_layers=num_hidden_layers, - num_attention_heads=num_attention_heads, - num_key_value_heads=num_key_value_heads, - hidden_act=hidden_act, - max_position_embeddings=max_position_embeddings, - initializer_range=initializer_range, - rms_norm_eps=rms_norm_eps, - use_cache=use_cache, - tie_word_embeddings=tie_word_embeddings, - rope_theta=rope_theta, - use_sliding_window=use_sliding_window, - sliding_window=sliding_window, - max_window_layers=max_window_layers, - attention_dropout=attention_dropout, - rope_scaling=rope_scaling, - ) - ) - self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings self.hidden_size = hidden_size diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 208c7ee97686..dca6d5d9eeef 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1086,14 +1086,6 @@ def __init__(self, config: Qwen2Config): super().__init__(config) self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size - if hasattr(config, "text_config"): - config = config.get_text_config() - config._attn_implementation = ( - config._attn_implementation if config._attn_implementation is not None else "eager" - ) - logger.warning_once( - "If you are loading Qwen2Model directly, make sure to load with `config.text_config` in the ipnut arguments" - ) self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) self.layers = nn.ModuleList( @@ -1433,11 +1425,9 @@ class Qwen2VLForConditionalGeneration(Qwen2VLPreTrainedModel, GenerationMixin): def __init__(self, config): super().__init__(config) self.visual = Qwen2VisionTransformerPretrainedModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) - self.model = Qwen2VLModel._from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] + config.vision_config, attn_implementation=config._attn_implementation ) + self.model = Qwen2VLModel._from_config(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) self.padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 43b83f85d78e..32e28852da1a 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -445,7 +445,6 @@ class BlipModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_pruning = False test_resize_embeddings = False test_attention_outputs = False - _is_composite = True def setUp(self): self.model_tester = BlipModelTester(self) @@ -470,10 +469,6 @@ def test_retain_grad_hidden_states_attentions(self): def test_model_get_set_embeddings(self): pass - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") - def test_sdpa_can_dispatch_composite_models(self): - pass - # override as the `logit_scale` parameter initilization is different for Blip def test_initialization(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -809,7 +804,6 @@ class BlipVQAModelTest(ModelTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - _is_composite = True def setUp(self): self.model_tester = BlipVQAModelTester(self) @@ -880,10 +874,6 @@ def test_inputs_embeds(self): def test_model_get_set_embeddings(self): pass - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") - def test_sdpa_can_dispatch_composite_models(self): - pass - @require_torch class BlipTextRetrievalModelTest(ModelTesterMixin, unittest.TestCase): @@ -894,7 +884,6 @@ class BlipTextRetrievalModelTest(ModelTesterMixin, unittest.TestCase): test_resize_embeddings = False test_attention_outputs = False test_torchscript = False - _is_composite = True def setUp(self): self.model_tester = BlipTextRetrievalModelTester(self) @@ -919,10 +908,6 @@ def test_retain_grad_hidden_states_attentions(self): def test_model_get_set_embeddings(self): pass - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") - def test_sdpa_can_dispatch_composite_models(self): - pass - def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index 4a5f67bbce12..2da7fe33b4f4 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -567,6 +567,10 @@ def test_model_from_pretrained(self): def test_sdpa_can_dispatch_composite_models(self): pass + @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") + def test_flash_attn_2_can_dispatch_composite_models(self): + pass + @unittest.skipIf(not is_torch_greater_or_equal_than_2_0, reason="pytorch 2.0 or higher is required") @require_torch diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index 00fafa29e8cc..30084c50e36c 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -512,6 +512,10 @@ def _create_and_check_torchscript(self, config, inputs_dict): def test_sdpa_can_dispatch_composite_models(self): pass + @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") + def test_flash_attn_2_can_dispatch_composite_models(self): + pass + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py index 9d0be00ea7bd..956243dccebe 100644 --- a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py +++ b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py @@ -226,7 +226,6 @@ class Qwen2VLModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCas all_generative_model_classes = (Qwen2VLForConditionalGeneration,) if is_torch_available() else () test_pruning = False test_head_masking = False - _is_composite = True def setUp(self): self.model_tester = Qwen2VLVisionText2TextModelTester(self) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index baa49a427fef..57f4dd51efcb 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4452,7 +4452,6 @@ def test_flash_attn_2_generate_use_cache(self): @require_flash_attn @require_torch_gpu @mark.flash_attn_test - @slow def test_flash_attn_2_can_dispatch_composite_models(self): """ Tests if composite models can dispatch on FA2 if the sub-models supports FA2. From 6d02e5c06daa5a8397bd2b7fb48e337226dd05a9 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 4 Oct 2024 11:39:18 +0200 Subject: [PATCH 55/68] some more fixes --- src/transformers/configuration_utils.py | 2 +- src/transformers/modeling_utils.py | 2 +- .../models/auto/configuration_auto.py | 1 - .../models/dbrx/configuration_dbrx.py | 74 ++++++++++--------- .../models/idefics2/modeling_idefics2.py | 2 - .../models/mllama/modeling_mllama.py | 2 - .../models/paligemma/modeling_paligemma.py | 1 - .../qwen2_audio/modeling_qwen2_audio.py | 2 - .../models/qwen2_vl/modeling_qwen2_vl.py | 5 +- .../video_llava/modeling_video_llava.py | 8 -- tests/models/blip_2/test_modeling_blip_2.py | 19 +++++ tests/models/idefics/test_modeling_idefics.py | 17 ++++- .../test_modeling_instructblip.py | 10 +++ .../test_modeling_instructblipvideo.py | 12 ++- tests/models/kosmos2/test_modeling_kosmos2.py | 23 +++++- 15 files changed, 121 insertions(+), 59 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index e39f6fc072c7..52d771ff98cc 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -771,7 +771,7 @@ def __repr__(self): return f"{self.__class__.__name__} {self.to_json_string()}" def __iter__(self): - for attr in copy.deepcopy(self.__dict__): + for attr in self.__dict__: yield attr def to_diff_dict(self) -> Dict[str, Any]: diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index f79b921b758b..32dc9adc9888 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1589,7 +1589,7 @@ def _autoset_attn_implementation( sub_configs = { key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) } - if sub_configs and all(name not in cls.__name__.lower() for name in ["dbrx"]): + if sub_configs and "dbrx" not in cls.__name__.lower(): attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): attn_implementation_per_subconfig[key] = ( diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index d24ac9846559..6d55f87d60ac 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -670,7 +670,6 @@ ("donut-swin", "donut"), ("kosmos-2", "kosmos2"), ("maskformer-swin", "maskformer"), - ("musicgen_melody_decoder", "musicgen_melody"), ("xclip", "x_clip"), ("clip_vision_model", "clip"), ("qwen2_audio_encoder", "qwen2_audio"), diff --git a/src/transformers/models/dbrx/configuration_dbrx.py b/src/transformers/models/dbrx/configuration_dbrx.py index dde5232ae5cc..1052ebbe8991 100644 --- a/src/transformers/models/dbrx/configuration_dbrx.py +++ b/src/transformers/models/dbrx/configuration_dbrx.py @@ -14,7 +14,8 @@ # limitations under the License. """DBRX model configuration""" -from typing import Any, Optional +import copy +from typing import Any, Dict, Optional from ...configuration_utils import PretrainedConfig from ...utils import logging @@ -23,7 +24,7 @@ logger = logging.get_logger(__name__) -class DbrxAttentionConfig(PretrainedConfig): +class DbrxAttentionConfig: """Configuration class for Dbrx Attention. [`DbrxAttention`] class. It is used to instantiate attention layers @@ -49,7 +50,6 @@ def __init__( rope_theta: float = 10000.0, **kwargs: Any, ): - super().__init__(**kwargs) self.attn_pdrop = attn_pdrop self.clip_qkv = clip_qkv self.kv_n_heads = kv_n_heads @@ -61,25 +61,20 @@ def __init__( if len(kwargs) != 0: raise ValueError(f"Found unknown {kwargs=}") - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any) -> "PretrainedConfig": - cls._set_token_in_kwargs(kwargs) + def to_dict(self) -> Dict[str, Any]: + """ + Serializes this instance to a Python dictionary. - config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + output = copy.deepcopy(self.__dict__) + if hasattr(self.__class__, "model_type"): + output["model_type"] = self.__class__.model_type + return output - if config_dict.get("model_type") == "dbrx": - config_dict = config_dict["attn_config"] - if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: - logger.warning( - f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " - + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." - ) - - return cls.from_dict(config_dict, **kwargs) - - -class DbrxFFNConfig(PretrainedConfig): +class DbrxFFNConfig: """Configuration class for Dbrx FFN. [`DbrxFFN`] class. It is used to instantiate feedforward layers according to @@ -128,22 +123,17 @@ def __init__( if len(kwargs) != 0: raise ValueError(f"Found unknown {kwargs=}") - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any) -> "PretrainedConfig": - cls._set_token_in_kwargs(kwargs) - - config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) - - if config_dict.get("model_type") == "dbrx": - config_dict = config_dict["ffn_config"] + def to_dict(self) -> Dict[str, Any]: + """ + Serializes this instance to a Python dictionary. - if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: - logger.warning( - f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " - + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." - ) - - return cls.from_dict(config_dict, **kwargs) + Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + output = copy.deepcopy(self.__dict__) + if hasattr(self.__class__, "model_type"): + output["model_type"] = self.__class__.model_type + return output class DbrxConfig(PretrainedConfig): @@ -256,3 +246,19 @@ def __init__( raise ValueError("tie_word_embeddings is not supported for DBRX models.") super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) + + def to_dict(self) -> Dict[str, Any]: + """ + Serializes this instance to a Python dictionary. + + Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + output = super().to_dict() + + for key, value in output.items(): + if key in ["ffn_config", "attn_config"]: + value = value.to_dict() + output[key] = value + + return output diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 4d9819cb18ac..d9dfcecf89a8 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -602,7 +602,6 @@ class Idefics2PreTrainedModel(PreTrainedModel): _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True _supports_cache_class = True - _is_composite = True def _init_weights(self, module): std = ( @@ -649,7 +648,6 @@ def _init_weights(self, module): ) class Idefics2VisionTransformer(Idefics2PreTrainedModel): _supports_sdpa = False - _is_composite = False def __init__(self, config: Idefics2VisionConfig): super().__init__(config) diff --git a/src/transformers/models/mllama/modeling_mllama.py b/src/transformers/models/mllama/modeling_mllama.py index 31d2bfe0c8e0..f3eba3725cd9 100644 --- a/src/transformers/models/mllama/modeling_mllama.py +++ b/src/transformers/models/mllama/modeling_mllama.py @@ -2032,8 +2032,6 @@ def prepare_inputs_for_generation( MLLAMA_START_DOCSTRING, ) class MllamaForConditionalGeneration(MllamaPreTrainedModel, GenerationMixin): - _is_composite = True - def __init__(self, config: MllamaConfig): super().__init__(config) self.vocab_size = config.text_config.vocab_size diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index 24cf89edc77b..e3588ce310a8 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -192,7 +192,6 @@ class PaliGemmaPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["PaliGemmaMultiModalProjector"] _skip_keys_device_placement = "past_key_values" - _supports_sdpa = True _supports_cache_class = True _supports_quantized_cache = True _supports_static_cache = True diff --git a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py index 9170fdb2052e..3bc2f71b8c38 100644 --- a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py +++ b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py @@ -543,8 +543,6 @@ class Qwen2AudioPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["Qwen2AudioAttention"] _skip_keys_device_placement = "past_key_values" - _supports_flash_attn_2 = True - _supports_sdpa = True def _init_weights(self, module): # important: this ported version of Qwen2Audio isn't meant for training from scratch - only diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index dca6d5d9eeef..85418a134aa1 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -49,7 +49,6 @@ logging, replace_return_docstrings, ) -from ..qwen2.configuration_qwen2 import Qwen2Config from .configuration_qwen2_vl import Qwen2VLConfig, Qwen2VLVisionConfig @@ -1082,7 +1081,7 @@ def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor) -> torch. QWEN2VL_START_DOCSTRING, ) class Qwen2VLModel(Qwen2VLPreTrainedModel): - def __init__(self, config: Qwen2Config): + def __init__(self, config: Qwen2VLConfig): super().__init__(config) self.padding_idx = config.pad_token_id self.vocab_size = config.vocab_size @@ -1427,7 +1426,7 @@ def __init__(self, config): self.visual = Qwen2VisionTransformerPretrainedModel._from_config( config.vision_config, attn_implementation=config._attn_implementation ) - self.model = Qwen2VLModel._from_config(config) + self.model = Qwen2VLModel(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) self.padding_side = "left" # set it to left by default, user can use setter to change padding_sides diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index 05e633d56e44..a9aae08f416d 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -147,14 +147,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - @property - def _supports_sdpa(self): - """ - Retrieve language_model's attribute to check whether the model supports - SDPA or not. - """ - return self.language_model._supports_sdpa - VIDEO_LLAVA_INPUTS_DOCSTRING = r""" Args: diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index aff94374d457..3abb63647f12 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -20,6 +20,7 @@ import numpy as np import requests +from parameterized import parameterized from transformers import CONFIG_MAPPING, Blip2Config, Blip2QFormerConfig, Blip2VisionConfig from transformers.testing_utils import ( @@ -493,6 +494,15 @@ def test_save_load_fast_init_to_base(self): def test_sdpa_can_dispatch_composite_models(self): pass + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + pass + + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_generate(self): + pass + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() @@ -771,6 +781,15 @@ def test_cpu_offload(self): def test_sdpa_can_dispatch_composite_models(self): pass + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + pass + + @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_generate(self): + pass + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index 2da7fe33b4f4..de7323ce54cf 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -16,12 +16,15 @@ import unittest +from parameterized import parameterized + from transformers import BitsAndBytesConfig, IdeficsConfig, is_torch_available, is_vision_available from transformers.testing_utils import ( TestCasePlus, is_pt_tf_cross_test, require_bitsandbytes, require_torch, + require_torch_sdpa, require_vision, slow, torch_device, @@ -309,6 +312,12 @@ def prepare_config_and_inputs_for_common(self): def prepare_pixel_values(self): return floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) + @require_torch_sdpa + @slow + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + self.skipTest(reason="Idefics has a hard requirement on SDPA, skipping this test") + @unittest.skipIf(not is_torch_greater_or_equal_than_2_0, reason="pytorch 2.0 or higher is required") @require_torch @@ -563,11 +572,17 @@ def test_model_from_pretrained(self): model = IdeficsModel.from_pretrained(model_name) self.assertIsNotNone(model) + @require_torch_sdpa + @slow + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + self.skipTest(reason="Idefics has a hard requirement on SDPA, skipping this test") + @unittest.skip("Idefics has a hard requirement on SDPA") def test_sdpa_can_dispatch_composite_models(self): pass - @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") + @unittest.skip("Idefics has a hard requirement on SDPA") def test_flash_attn_2_can_dispatch_composite_models(self): pass diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 37c5fac8aec4..d18653974d04 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -20,6 +20,7 @@ import numpy as np import requests +from parameterized import parameterized from transformers import ( CONFIG_MAPPING, @@ -534,6 +535,15 @@ def test_model_from_pretrained(self): def test_sdpa_can_dispatch_composite_models(self): pass + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + pass + + @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_generate(self): + pass + # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 589bb5e7629c..730ee9c2d868 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -20,6 +20,7 @@ import numpy as np from huggingface_hub import hf_hub_download +from parameterized import parameterized from transformers import ( CONFIG_MAPPING, @@ -551,10 +552,19 @@ def test_model_from_pretrained(self): model = InstructBlipVideoForConditionalGeneration.from_pretrained(model_name) self.assertIsNotNone(model) - @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") + @unittest.skip("InstructBlipvideo doesn't support SPDA with this particulr LM bacbone") def test_sdpa_can_dispatch_composite_models(self): pass + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @unittest.skip("InstructBlipvideo doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + pass + + @unittest.skip("InstructBlipvideo doesn't support SPDA with this particulr LM bacbone") + def test_eager_matches_sdpa_generate(self): + pass + # We will verify our results on an image of cute cats def prepare_video(): diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index 30084c50e36c..40dacb3db698 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -22,11 +22,21 @@ import numpy as np import requests +from parameterized import parameterized from transformers import AutoModelForVision2Seq, AutoProcessor, Kosmos2Config from transformers.models.kosmos2.configuration_kosmos2 import Kosmos2TextConfig, Kosmos2VisionConfig -from transformers.testing_utils import IS_ROCM_SYSTEM, require_torch, require_vision, slow, torch_device -from transformers.utils import is_torch_available, is_vision_available +from transformers.testing_utils import ( + IS_ROCM_SYSTEM, + require_torch, + require_vision, + slow, + torch_device, +) +from transformers.utils import ( + is_torch_available, + is_vision_available, +) from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( @@ -508,6 +518,15 @@ def _create_and_check_torchscript(self, config, inputs_dict): # (Even with this call, there are still memory leak by ~0.04MB) self.clear_torch_jit_class_registry() + @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) + @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") + def test_eager_matches_sdpa_inference(self, torch_dtype: str): + pass + + @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") + def test_eager_matches_sdpa_generate(self): + pass + @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") def test_sdpa_can_dispatch_composite_models(self): pass From a578fdc985dc7f681d2cd64dcefab479422c0d12 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 4 Oct 2024 11:54:20 +0200 Subject: [PATCH 56/68] update --- tests/models/blip/test_modeling_blip.py | 5 --- .../test_modeling_encoder_decoder.py | 34 ++--------------- tests/models/gemma2/test_modeling_gemma2.py | 3 +- tests/models/mamba2/test_modeling_mamba2.py | 10 +++-- .../models/musicgen/test_modeling_musicgen.py | 10 ++--- .../test_modeling_musicgen_melody.py | 4 +- .../test_modeling_recurrent_gemma.py | 12 +++--- .../test_modeling_speech_encoder_decoder.py | 38 ++----------------- .../test_modeling_vision_encoder_decoder.py | 32 ++-------------- 9 files changed, 31 insertions(+), 117 deletions(-) diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 32e28852da1a..2f8ee3229ff2 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -1113,11 +1113,6 @@ class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False test_torchscript = False - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because vision models used in tests don't support SDPA - supports_sdpa = False - def setUp(self): self.model_tester = BlipTextImageModelsModelTester(self) diff --git a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py index e8b47f35a67d..1683e4773553 100644 --- a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py +++ b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py @@ -17,8 +17,6 @@ import tempfile import unittest -from parameterized import parameterized - from transformers import is_torch_available, logging from transformers.testing_utils import ( CaptureLogger, @@ -28,7 +26,6 @@ slow, torch_device, ) -from transformers.utils import is_torch_bf16_available_on_device, is_torch_fp16_available_on_device from ...test_modeling_common import ids_tensor from ..bart.test_modeling_bart import BartStandaloneDecoderModelTester @@ -682,29 +679,11 @@ def test_real_model_save_load_from_pretrained(self): max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow - def test_eager_matches_sdpa_inference(self, torch_dtype: str): + def test_sdpa_can_dispatch_composite_models(self): if not self.supports_sdpa: self.skipTest("SDPA is not supported") - if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): - self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") - - if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): - self.skipTest( - f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" - ) - - # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. - if torch_dtype == "float16": - torch_dtype = torch.float16 - elif torch_dtype == "bfloat16": - torch_dtype = torch.bfloat16 - elif torch_dtype == "float32": - torch_dtype = torch.float32 - inputs_dict = self.prepare_config_and_inputs() encoder_config, decoder_config = inputs_dict["config"], inputs_dict["decoder_config"] config = EncoderDecoderConfig.from_encoder_decoder_configs( @@ -714,7 +693,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) - model_sdpa = EncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = EncoderDecoderModel.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 @@ -730,9 +709,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): # If the model supports sdpa (i.e. all of sub-models supports it) we'll dispatch safely # Otherwise we should raise error that SDPA is not supported, as some of the sub-models doesn't support if encoder_attn == "sdpa" and decoder_attn == "sdpa": - model_sdpa_explicit = EncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) + model_sdpa_explicit = EncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) self.assertTrue( @@ -741,13 +718,10 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): ) else: with self.assertRaises(ValueError): - model_sdpa_explicit = EncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) + model_sdpa_explicit = EncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") model_eager = EncoderDecoderModel.from_pretrained( tmpdirname, - torch_dtype=torch_dtype, attn_implementation="eager", ) model_eager = model_eager.eval().to(torch_device) diff --git a/tests/models/gemma2/test_modeling_gemma2.py b/tests/models/gemma2/test_modeling_gemma2.py index b8d44fb10da1..ff72902a7119 100644 --- a/tests/models/gemma2/test_modeling_gemma2.py +++ b/tests/models/gemma2/test_modeling_gemma2.py @@ -86,8 +86,7 @@ def setUp(self): def test_model_outputs_equivalence(self, **kwargs): pass - unittest.skip("Gemma2's forcefully disables sdpa due to softcapping") - + @unittest.skip("Gemma2's forcefully disables sdpa due to softcapping") def test_sdpa_can_dispatch_non_composite_models(self): pass diff --git a/tests/models/mamba2/test_modeling_mamba2.py b/tests/models/mamba2/test_modeling_mamba2.py index c6488d473c0d..f19358a22f4b 100644 --- a/tests/models/mamba2/test_modeling_mamba2.py +++ b/tests/models/mamba2/test_modeling_mamba2.py @@ -207,10 +207,6 @@ def test_generate_without_input_ids(self): def test_generate_from_inputs_embeds_decoder_only(self): pass - @unittest.skip(reason="To fix, Mamba 2 cache slicing test case is an edge case") - def test_inputs_embeds_matches_input_ids_with_generate(self): - pass - @unittest.skip(reason="To fix, Mamba 2 cache slicing test case is an edge case") def test_greedy_generate_dict_outputs_use_cache(self): pass @@ -279,6 +275,12 @@ def recursive_check(tuple_object, dict_object): dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) check_equivalence(model, tuple_inputs, dict_inputs, {"output_hidden_states": True}) + @unittest.skip( + reason="Mamba2 does not support generating with input embeddings (custom cache_position computation)" + ) + def test_inputs_embeds_matches_input_ids_with_generate(self): + pass + @require_torch @slow diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 0aeedc9b7b65..2cc4aff5c043 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -270,10 +270,6 @@ def test_model_get_set_embeddings(self): def test_inputs_embeds_matches_input_ids(self): pass - @unittest.skip(reason="MusicGen does not use inputs_embeds") - def test_inputs_embeds_matches_input_ids_with_generate(self): - pass - @unittest.skip(reason="MusicGen does not support all arguments tested") def test_model_outputs_equivalence(self): pass @@ -628,11 +624,12 @@ def test_flash_attn_2_generate_use_cache(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow + # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa: + if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -1978,11 +1975,12 @@ def test_sdpa_can_dispatch_composite_models(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow + # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa: + if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index 7398c58ab4d6..7183f5917d4c 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -626,11 +626,12 @@ def test_flash_attn_2_generate_use_cache(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow + # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa: + if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -1958,6 +1959,7 @@ def test_sdpa_can_dispatch_composite_models(self): @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow + # Copied from tests.test_modeling_common.ModelTesterMixin.test_eager_matches_sdpa_inference def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") diff --git a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py index f2c6c1ddb1c3..d2f658f56bd8 100644 --- a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py +++ b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py @@ -103,7 +103,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = torch.tril(torch.ones(self.batch_size, self.seq_length)).to(torch_device) + input_mask = torch.tril(torch.ones_like(input_ids).to(torch_device)) token_type_ids = None if self.use_token_type_ids: @@ -390,12 +390,6 @@ def test_left_padding_compatibility(self): def test_assisted_decoding_sample(self): pass - @unittest.skip( - reason="RecurentGemma generation tests are not fully supported" - ) # TODO: @gante after adding MixinTests - def test_inputs_embeds_matches_input_ids_with_generate(self): - pass - def _check_hidden_states_for_generate( self, batch_size, hidden_states, min_length, max_length, config, use_cache=False, num_beam_groups=1 ): @@ -419,6 +413,10 @@ def _check_hidden_states_for_generate( def test_initialization(self): pass + @unittest.skip(reason="RecurrentGemma does not support generating with input embeddings (missing position_ids)") + def test_inputs_embeds_matches_input_ids_with_generate(self): + pass + @require_torch_accelerator @slow diff --git a/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py b/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py index e81210ab7914..bf4697afaed2 100644 --- a/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py +++ b/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py @@ -17,8 +17,6 @@ import tempfile import unittest -from parameterized import parameterized - from transformers import is_torch_available from transformers.testing_utils import ( require_deterministic_for_xpu, @@ -27,10 +25,6 @@ slow, torch_device, ) -from transformers.utils import ( - is_torch_bf16_available_on_device, - is_torch_fp16_available_on_device, -) from ...test_modeling_common import floats_tensor, ids_tensor, random_attention_mask from ..bert.test_modeling_bert import BertModelTester @@ -453,29 +447,8 @@ def test_real_model_save_load_from_pretrained(self): max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - # if not self.supports_sdpa: - # self.skipTest("SDPA is not supported") - - if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): - self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") - - if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): - self.skipTest( - f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" - ) - - # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. - if torch_dtype == "float16": - torch_dtype = torch.float16 - elif torch_dtype == "bfloat16": - torch_dtype = torch.bfloat16 - elif torch_dtype == "float32": - torch_dtype = torch.float32 - + def test_sdpa_can_dispatch_composite_models(self): inputs_dict = self.prepare_config_and_inputs() encoder_config, decoder_config = inputs_dict["config"], inputs_dict["decoder_config"] config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs( @@ -485,7 +458,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) - model_sdpa = SpeechEncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = SpeechEncoderDecoderModel.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 @@ -501,9 +474,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): # If the model supports sdpa (i.e. all of sub-models supports it) we'll dispatch safely # Otherwise we should raise error that SDPA is not supported, as some of the sub-models doesn't support if encoder_attn == "sdpa" and decoder_attn == "sdpa": - model_sdpa_explicit = SpeechEncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) + model_sdpa_explicit = SpeechEncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) self.assertTrue( @@ -513,12 +484,11 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): else: with self.assertRaises(ValueError): model_sdpa_explicit = SpeechEncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + tmpdirname, attn_implementation="sdpa" ) model_eager = SpeechEncoderDecoderModel.from_pretrained( tmpdirname, - torch_dtype=torch_dtype, attn_implementation="eager", ) model_eager = model_eager.eval().to(torch_device) diff --git a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py index 8593f0494191..bbad7aa5cea4 100644 --- a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py +++ b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py @@ -20,7 +20,6 @@ from datasets import load_dataset from huggingface_hub import hf_hub_download from packaging import version -from parameterized import parameterized from transformers import DonutProcessor, NougatProcessor, TrOCRProcessor from transformers.testing_utils import ( @@ -37,8 +36,6 @@ from transformers.utils import ( cached_property, is_torch_available, - is_torch_bf16_available_on_device, - is_torch_fp16_available_on_device, is_vision_available, ) @@ -388,29 +385,11 @@ def test_real_model_save_load_from_pretrained(self): max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa - @slow - def test_eager_matches_sdpa_inference(self, torch_dtype: str): + def test_sdpa_can_dispatch_composite_models(self): if not self.supports_sdpa: self.skipTest("SDPA is not supported") - if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): - self.skipTest(f"float16 not supported on {torch_device} (on the specific device currently used)") - - if torch_dtype == "bfloat16" and not is_torch_bf16_available_on_device(torch_device): - self.skipTest( - f"bfloat16 not supported on {torch_device} (on the specific device currently used, e.g. Nvidia T4 GPU)" - ) - - # Not sure whether it's fine to put torch.XXX in a decorator if torch is not available so hacking it here instead. - if torch_dtype == "float16": - torch_dtype = torch.float16 - elif torch_dtype == "bfloat16": - torch_dtype = torch.bfloat16 - elif torch_dtype == "float32": - torch_dtype = torch.float32 - inputs_dict = self.prepare_config_and_inputs() encoder_config, decoder_config = inputs_dict["config"], inputs_dict["decoder_config"] config = VisionEncoderDecoderConfig.from_encoder_decoder_configs( @@ -420,7 +399,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) - model_sdpa = VisionEncoderDecoderModel.from_pretrained(tmpdirname, torch_dtype=torch_dtype) + model_sdpa = VisionEncoderDecoderModel.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 @@ -436,9 +415,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): # If the model supports sdpa (i.e. all of sub-models supports it) we'll dispatch safely # Otherwise we should raise error that SDPA is not supported, as some of the sub-models doesn't support if encoder_attn == "sdpa" and decoder_attn == "sdpa": - model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" - ) + model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) self.assertTrue( @@ -448,12 +425,11 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): else: with self.assertRaises(ValueError): model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( - tmpdirname, torch_dtype=torch_dtype, attn_implementation="sdpa" + tmpdirname, attn_implementation="sdpa" ) model_eager = VisionEncoderDecoderModel.from_pretrained( tmpdirname, - torch_dtype=torch_dtype, attn_implementation="eager", ) model_eager = model_eager.eval().to(torch_device) From c02a943969b3d781e7945c6dc6bc2e9a173de000 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 4 Oct 2024 13:49:48 +0200 Subject: [PATCH 57/68] more updates --- src/transformers/modeling_utils.py | 2 +- .../models/idefics/modeling_idefics.py | 42 ++++++++++++++++ .../models/idefics2/configuration_idefics2.py | 15 ++++++ .../models/idefics2/modeling_idefics2.py | 50 +++++++++---------- .../models/idefics2/test_modeling_idefics2.py | 3 ++ 5 files changed, 85 insertions(+), 27 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 32dc9adc9888..fa37d7a04346 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1589,7 +1589,7 @@ def _autoset_attn_implementation( sub_configs = { key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) } - if sub_configs and "dbrx" not in cls.__name__.lower(): + if sub_configs: attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): attn_implementation_per_subconfig[key] = ( diff --git a/src/transformers/models/idefics/modeling_idefics.py b/src/transformers/models/idefics/modeling_idefics.py index 9ace74f03893..f9de2ff571c2 100644 --- a/src/transformers/models/idefics/modeling_idefics.py +++ b/src/transformers/models/idefics/modeling_idefics.py @@ -980,6 +980,48 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() + # Adapted from transformers.modeling_utils.PreTrainedModel._autoset_attn_implementation + @classmethod + def _autoset_attn_implementation( + cls, + config, + use_flash_attention_2: bool = False, + torch_dtype: Optional[torch.dtype] = None, + device_map: Optional[Union[str, Dict[str, int]]] = None, + check_device_map: bool = True, + ): + requested_attn_implementation = None + if hasattr(config, "_attn_implementation_internal") and config._attn_implementation_internal is not None: + if config._attn_implementation != "flash_attention_2" and use_flash_attention_2: + raise ValueError( + f'Both attn_implementation="{config._attn_implementation}" and `use_flash_attention_2=True` were used when loading the model, which are not compatible.' + ' We recommend to just use `attn_implementation="flash_attention_2"` when loading the model.' + ) + + if not isinstance(config._attn_implementation, dict) and config._attn_implementation not in [ + "eager", + "sdpa", + "flash_attention_2", + ]: + message = f'Specified `attn_implementation="{config._attn_implementation}"` is not supported. The only possible arguments are `attn_implementation="eager"` (manual attention implementation)' + if cls._supports_flash_attn_2: + message += ', `"attn_implementation=flash_attention_2"` (implementation using flash attention 2)' + if cls._supports_sdpa: + message += ', `"attn_implementation=sdpa"` (implementation using torch.nn.functional.scaled_dot_product_attention)' + raise ValueError(message + ".") + + # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available. + requested_attn_implementation = config._attn_implementation_internal + + # IDEFICS has ahrd requirement on SDPA + if requested_attn_implementation not in ["sdpa", None]: + logger.warning_once( + f"Idefics supports only SDPA attention, but the model being loaded with {requested_attn_implementation} " + "Falling back to SDPA. If you need other attention implementations, please open an issue." + ) + config._attn_implementation = "sdpa" + return config + LLAMA_INPUTS_DOCSTRING = r""" Args: diff --git a/src/transformers/models/idefics2/configuration_idefics2.py b/src/transformers/models/idefics2/configuration_idefics2.py index 69bdcf6d2bce..64743d1cd470 100644 --- a/src/transformers/models/idefics2/configuration_idefics2.py +++ b/src/transformers/models/idefics2/configuration_idefics2.py @@ -134,6 +134,10 @@ class Idefics2PerceiverConfig(PretrainedConfig): Args: hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): The non-linear activation function (function or string) in the perceiver block. + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. resampler_n_latents (`int`, *optional*, defaults to 64): Number of latent embeddings to resample ("compress") the input sequence to (usually < 128). resampler_depth (`int`, *optional*, defaults to 3): @@ -153,6 +157,8 @@ class Idefics2PerceiverConfig(PretrainedConfig): def __init__( self, hidden_act="silu", + hidden_size=4096, + rms_norm_eps=1e-06, resampler_n_latents=64, resampler_depth=3, resampler_n_heads=16, @@ -162,6 +168,8 @@ def __init__( **kwargs, ): self.hidden_act = hidden_act + self.hidden_size = hidden_size + self.rms_norm_eps = rms_norm_eps self.resampler_n_latents = resampler_n_latents self.resampler_depth = resampler_depth self.resampler_n_heads = resampler_n_heads @@ -258,5 +266,12 @@ def __init__( ) self.text_config = text_config + if self.text_config.hidden_size != self.perceiver_config.hidden_size: + self.perceiver_config.hidden_size = self.text_config.hidden_size + self.perceiver_config.rms_norm_eps = self.text_config.rms_norm_eps + logger.warning_once( + "Perceiver config has a different `hidden_size` than text config, which means default values were used. " + "In your model's config on the hub, add `hidden_size` and `rms_norm_eps` keys under the `perceiver_config` dict. " + ) super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings) diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index d9dfcecf89a8..2f3d3f3c5d8c 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -39,7 +39,7 @@ replace_return_docstrings, ) from ..auto import AutoModel -from .configuration_idefics2 import Idefics2Config, Idefics2VisionConfig +from .configuration_idefics2 import Idefics2Config, Idefics2PerceiverConfig, Idefics2VisionConfig if is_flash_attn_2_available(): @@ -763,12 +763,12 @@ def __init__(self, config, layer_idx: Optional[int] = None) -> None: super().__init__() self.layer_idx = None - self.hidden_size = config.text_config.hidden_size - self.num_heads = config.perceiver_config.resampler_n_heads - self.head_dim = config.perceiver_config.resampler_head_dim - self.num_key_value_heads = config.perceiver_config.num_key_value_heads + self.hidden_size = config.hidden_size + self.num_heads = config.resampler_n_heads + self.head_dim = config.resampler_head_dim + self.num_key_value_heads = config.num_key_value_heads self.num_key_value_groups = self.num_heads // self.num_key_value_heads - self.attention_dropout = config.perceiver_config.attention_dropout + self.attention_dropout = config.attention_dropout self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) @@ -994,25 +994,20 @@ def forward( class Idefics2PerceiverLayer(nn.Module): def __init__(self, config, layer_idx: int): super().__init__() - self.hidden_size = config.text_config.hidden_size - self.n_latents = config.perceiver_config.resampler_n_latents - self.depth = config.perceiver_config.resampler_depth - self.rms_norm_eps = config.text_config.rms_norm_eps - attn_implementation = ( - config._attn_implementation["perceiver_config"] - if config._attn_implementation["perceiver_config"] is not None - else "eager" - ) + self.hidden_size = config.hidden_size + self.n_latents = config.resampler_n_latents + self.depth = config.resampler_depth + self.rms_norm_eps = config.rms_norm_eps self.input_latents_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.input_context_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) - self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[attn_implementation](config, layer_idx=layer_idx) + self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx) self.post_attention_layernorm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps) self.mlp = Idefics2MLP( - hidden_size=config.text_config.hidden_size, - intermediate_size=config.text_config.hidden_size * 4, - output_size=config.text_config.hidden_size, - hidden_act=config.perceiver_config.hidden_act, + hidden_size=config.hidden_size, + intermediate_size=config.hidden_size * 4, + output_size=config.hidden_size, + hidden_act=config.hidden_act, ) def forward( @@ -1090,14 +1085,15 @@ def forward( ) class Idefics2PerceiverResampler(Idefics2PreTrainedModel): _supports_sdpa = False + config_class = Idefics2PerceiverConfig def __init__(self, config) -> None: super().__init__(config) - self.hidden_size = config.text_config.hidden_size - self.hidden_act = config.perceiver_config.hidden_act - self.n_latents = config.perceiver_config.resampler_n_latents - self.depth = config.perceiver_config.resampler_depth - self.rms_norm_eps = config.text_config.rms_norm_eps + self.hidden_size = config.hidden_size + self.hidden_act = config.hidden_act + self.n_latents = config.resampler_n_latents + self.depth = config.resampler_depth + self.rms_norm_eps = config.rms_norm_eps # Create Latents for Perceiver self.latents = nn.Parameter(torch.ones(self.n_latents, self.hidden_size)) @@ -1154,7 +1150,9 @@ def __init__(self, config): output_size=config.text_config.hidden_size, hidden_act=config.text_config.hidden_act, ) - self.perceiver_resampler = Idefics2PerceiverResampler(config) + self.perceiver_resampler = Idefics2PerceiverResampler._from_config( + config.perceiver_config, attn_implementation=config._attn_implementation["perceiver_config"] + ) def forward(self, image_hidden_states, attention_mask): image_hidden_states = self.modality_projection(image_hidden_states) diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index c2f5e712e7c3..075f74f5c724 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -341,11 +341,13 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_sdpa.eval().to(torch_device) vision_attn = None if model.vision_model._supports_sdpa else "eager" + perceiver_attn = None if model.connector.perceiver_resampler._supports_sdpa else "eager" self.assertTrue( model_sdpa.config._attn_implementation == {"text_config": None, "perceiver_config": None, "vision_config": None} ) self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.connector.perceiver_resampler.config._attn_implementation == perceiver_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) @@ -354,6 +356,7 @@ def test_sdpa_can_dispatch_composite_models(self): == {"text_config": "eager", "perceiver_config": "eager", "vision_config": "eager"} ) self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_sdpa.connector.perceiver_resampler.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ From 19de5950e7defc58ae13cea65321bd6356809da7 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 4 Oct 2024 15:11:43 +0200 Subject: [PATCH 58/68] update --- src/transformers/models/rag/modeling_rag.py | 4 +- tests/models/blip_2/test_modeling_blip_2.py | 146 +++++++++++++++--- tests/models/idefics/test_modeling_idefics.py | 15 +- .../test_modeling_instructblip.py | 74 +++++++-- .../test_modeling_instructblipvideo.py | 74 +++++++-- tests/models/kosmos2/test_modeling_kosmos2.py | 10 -- tests/test_modeling_common.py | 29 +++- 7 files changed, 281 insertions(+), 71 deletions(-) diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py index bc375b68e947..6ca6b8901f07 100644 --- a/src/transformers/models/rag/modeling_rag.py +++ b/src/transformers/models/rag/modeling_rag.py @@ -507,14 +507,14 @@ def __init__( from ..auto.modeling_auto import AutoModel question_encoder = AutoModel.from_config( - config.question_encoder, attn_implementation=config._attn_implementation + config.question_encoder, attn_implementation=config._attn_implementation["question_encoder"] ) if generator is None: from ..auto.modeling_auto import AutoModelForSeq2SeqLM generator = AutoModelForSeq2SeqLM.from_config( - config.generator, attn_implementation=config._attn_implementation + config.generator, attn_implementation=config._attn_implementation["generator"] ) self.retriever = retriever diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 3abb63647f12..80e8a3ff0e11 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -20,7 +20,6 @@ import numpy as np import requests -from parameterized import parameterized from transformers import CONFIG_MAPPING, Blip2Config, Blip2QFormerConfig, Blip2VisionConfig from transformers.testing_utils import ( @@ -28,6 +27,7 @@ require_torch_fp16, require_torch_gpu, require_torch_multi_accelerator, + require_torch_sdpa, require_vision, slow, torch_device, @@ -490,18 +490,72 @@ def test_save_load_fast_init_from_base(self): def test_save_load_fast_init_to_base(self): pass - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + @require_torch_sdpa def test_sdpa_can_dispatch_composite_models(self): - pass + """ + Tests if composite models dispatch correctly on SDPA/eager when requested so when loading the model. + This tests only by looking at layer names, as usually SDPA layers are calles "SDPAAttention". + In contrast to the above test, this one checks if the "config._attn_implamentation" is a dict after the model + is loaded, because we manually replicate requested attn implementation on each sub-config when loading. + See https://github.com/huggingface/transformers/pull/32238 for more info + + The test tries to cover most general cases of composite models, VLMs with vision and text configs. Any model + that has a different set of sub-configs has to overwrite this test. + """ + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - pass + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_generate(self): - pass + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" + vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" + qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue( + model_sdpa.config._attn_implementation + == {"text_config": None, "vision_config": None, "qformer_config": None} + ) + self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) + self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) + self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + self.assertTrue( + model_eager.config._attn_implementation + == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} + ) + self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and any( + module_attn == "sdpa" for module_attn in [text_attn, vision_attn, qformer_attn] + ): + raise ValueError("The SDPA model should have SDPA attention layers") def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() @@ -777,18 +831,72 @@ def test_save_load_fast_init_to_base(self): def test_cpu_offload(self): pass - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") + @require_torch_sdpa def test_sdpa_can_dispatch_composite_models(self): - pass + """ + Tests if composite models dispatch correctly on SDPA/eager when requested so when loading the model. + This tests only by looking at layer names, as usually SDPA layers are calles "SDPAAttention". + In contrast to the above test, this one checks if the "config._attn_implamentation" is a dict after the model + is loaded, because we manually replicate requested attn implementation on each sub-config when loading. + See https://github.com/huggingface/transformers/pull/32238 for more info + + The test tries to cover most general cases of composite models, VLMs with vision and text configs. Any model + that has a different set of sub-configs has to overwrite this test. + """ + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - pass + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) - @unittest.skip("Blip doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_generate(self): - pass + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" + vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" + qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue( + model_sdpa.config._attn_implementation + == {"text_config": None, "vision_config": None, "qformer_config": None} + ) + self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) + self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) + self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + self.assertTrue( + model_eager.config._attn_implementation + == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} + ) + self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and any( + module_attn == "sdpa" for module_attn in [text_attn, vision_attn, qformer_attn] + ): + raise ValueError("The SDPA model should have SDPA attention layers") def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/models/idefics/test_modeling_idefics.py b/tests/models/idefics/test_modeling_idefics.py index de7323ce54cf..00c21982c2b1 100644 --- a/tests/models/idefics/test_modeling_idefics.py +++ b/tests/models/idefics/test_modeling_idefics.py @@ -327,7 +327,6 @@ class IdeficsModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_pruning = False test_headmasking = False test_torchscript = False - _is_composite = True def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) @@ -572,18 +571,8 @@ def test_model_from_pretrained(self): model = IdeficsModel.from_pretrained(model_name) self.assertIsNotNone(model) - @require_torch_sdpa - @slow - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - self.skipTest(reason="Idefics has a hard requirement on SDPA, skipping this test") - - @unittest.skip("Idefics has a hard requirement on SDPA") - def test_sdpa_can_dispatch_composite_models(self): - pass - @unittest.skip("Idefics has a hard requirement on SDPA") - def test_flash_attn_2_can_dispatch_composite_models(self): + def test_sdpa_can_dispatch_non_composite_models(self): pass @@ -624,7 +613,7 @@ def test_training_gradient_checkpointing_use_reentrant_false(self): pass @unittest.skip("Idefics has a hard requirement on SDPA") - def test_sdpa_can_dispatch_composite_models(self): + def test_sdpa_can_dispatch_non_composite_models(self): pass diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index d18653974d04..056d2d8c75c7 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -20,7 +20,6 @@ import numpy as np import requests -from parameterized import parameterized from transformers import ( CONFIG_MAPPING, @@ -33,6 +32,7 @@ require_accelerate, require_bitsandbytes, require_torch, + require_torch_sdpa, require_vision, slow, torch_device, @@ -531,18 +531,72 @@ def test_model_from_pretrained(self): model = InstructBlipForConditionalGeneration.from_pretrained(model_name) self.assertIsNotNone(model) - @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") + @require_torch_sdpa def test_sdpa_can_dispatch_composite_models(self): - pass + """ + Tests if composite models dispatch correctly on SDPA/eager when requested so when loading the model. + This tests only by looking at layer names, as usually SDPA layers are calles "SDPAAttention". + In contrast to the above test, this one checks if the "config._attn_implamentation" is a dict after the model + is loaded, because we manually replicate requested attn implementation on each sub-config when loading. + See https://github.com/huggingface/transformers/pull/32238 for more info + + The test tries to cover most general cases of composite models, VLMs with vision and text configs. Any model + that has a different set of sub-configs has to overwrite this test. + """ + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - pass + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) - @unittest.skip("InstructBlip doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_generate(self): - pass + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" + vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" + qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue( + model_sdpa.config._attn_implementation + == {"text_config": None, "vision_config": None, "qformer_config": None} + ) + self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) + self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) + self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + self.assertTrue( + model_eager.config._attn_implementation + == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} + ) + self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and any( + module_attn == "sdpa" for module_attn in [text_attn, vision_attn, qformer_attn] + ): + raise ValueError("The SDPA model should have SDPA attention layers") # We will verify our results on an image of cute cats diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 730ee9c2d868..08f13fdaa6bb 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -20,7 +20,6 @@ import numpy as np from huggingface_hub import hf_hub_download -from parameterized import parameterized from transformers import ( CONFIG_MAPPING, @@ -33,6 +32,7 @@ require_accelerate, require_bitsandbytes, require_torch, + require_torch_sdpa, require_vision, slow, torch_device, @@ -552,18 +552,72 @@ def test_model_from_pretrained(self): model = InstructBlipVideoForConditionalGeneration.from_pretrained(model_name) self.assertIsNotNone(model) - @unittest.skip("InstructBlipvideo doesn't support SPDA with this particulr LM bacbone") + @require_torch_sdpa def test_sdpa_can_dispatch_composite_models(self): - pass + """ + Tests if composite models dispatch correctly on SDPA/eager when requested so when loading the model. + This tests only by looking at layer names, as usually SDPA layers are calles "SDPAAttention". + In contrast to the above test, this one checks if the "config._attn_implamentation" is a dict after the model + is loaded, because we manually replicate requested attn implementation on each sub-config when loading. + See https://github.com/huggingface/transformers/pull/32238 for more info + + The test tries to cover most general cases of composite models, VLMs with vision and text configs. Any model + that has a different set of sub-configs has to overwrite this test. + """ + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + if not self._is_composite: + self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @unittest.skip("InstructBlipvideo doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - pass + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + model = model_class(config) - @unittest.skip("InstructBlipvideo doesn't support SPDA with this particulr LM bacbone") - def test_eager_matches_sdpa_generate(self): - pass + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model_sdpa = model_class.from_pretrained(tmpdirname) + model_sdpa = model_sdpa.eval().to(torch_device) + + text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" + vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" + qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + + # `None` as it is the requested one which will be assigned to each sub-config + # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) + self.assertTrue( + model_sdpa.config._attn_implementation + == {"text_config": None, "vision_config": None, "qformer_config": None} + ) + self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) + self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) + self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + + model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") + model_eager = model_eager.eval().to(torch_device) + self.assertTrue( + model_eager.config._attn_implementation + == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} + ) + self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") + self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + + for name, submodule in model_eager.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + raise ValueError("The eager model should not have SDPA attention layers") + + has_sdpa = False + for name, submodule in model_sdpa.named_modules(): + class_name = submodule.__class__.__name__ + if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: + has_sdpa = True + break + if not has_sdpa and any( + module_attn == "sdpa" for module_attn in [text_attn, vision_attn, qformer_attn] + ): + raise ValueError("The SDPA model should have SDPA attention layers") # We will verify our results on an image of cute cats diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index 40dacb3db698..0d430ab3d725 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -22,7 +22,6 @@ import numpy as np import requests -from parameterized import parameterized from transformers import AutoModelForVision2Seq, AutoProcessor, Kosmos2Config from transformers.models.kosmos2.configuration_kosmos2 import Kosmos2TextConfig, Kosmos2VisionConfig @@ -518,15 +517,6 @@ def _create_and_check_torchscript(self, config, inputs_dict): # (Even with this call, there are still memory leak by ~0.04MB) self.clear_torch_jit_class_registry() - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) - @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") - def test_eager_matches_sdpa_inference(self, torch_dtype: str): - pass - - @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") - def test_eager_matches_sdpa_generate(self): - pass - @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") def test_sdpa_can_dispatch_composite_models(self): pass diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 57f4dd51efcb..56f9754a2009 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3794,7 +3794,7 @@ def test_attn_implementation_composite_models(self): } # set eager as it will be the one supported in all models - # we just need to test if passing a dict 'attn_implementation' fails or not + # we just need to test if passing 'attn_implementation' as a dict fails or not attn_implementation_per_subconfig = {} for key, sub_config in sub_configs.items(): attn_implementation_per_subconfig[key] = "eager" @@ -3813,6 +3813,10 @@ def test_attn_implementation_composite_models(self): @require_torch_sdpa def test_sdpa_can_dispatch_non_composite_models(self): + """ + Tests if non-composite models dispatch correctly on SDPA/eager when requested so when loading the model. + This tests only by looking at layer names, as usually SDPA layers are calles "SDPAAttention". + """ if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") @@ -3850,6 +3854,16 @@ def test_sdpa_can_dispatch_non_composite_models(self): @require_torch_sdpa def test_sdpa_can_dispatch_composite_models(self): + """ + Tests if composite models dispatch correctly on SDPA/eager when requested so when loading the model. + This tests only by looking at layer names, as usually SDPA layers are calles "SDPAAttention". + In contrast to the above test, this one checks if the "config._attn_implamentation" is a dict after the model + is loaded, because we manually replicate requested attn implementation on each sub-config when loading. + See https://github.com/huggingface/transformers/pull/32238 for more info + + The test tries to cover most general cases of composite models, VLMs with vision and text configs. Any model + that has a different set of sub-configs has to overwrite this test. + """ if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") @@ -3900,7 +3914,7 @@ def test_sdpa_can_dispatch_composite_models(self): if "SdpaAttention" in class_name or "SdpaSelfAttention" in class_name: has_sdpa = True break - if not has_sdpa and model_sdpa.config.model_type != "falcon": + if not has_sdpa and any(module_attn == "sdpa" for module_attn in [text_attn, vision_attn]): raise ValueError("The SDPA model should have SDPA attention layers") @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @@ -4200,7 +4214,7 @@ def test_sdpa_can_dispatch_on_flash(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if not self._is_composite and not model_class._supports_sdpa: + if not model_class._supports_sdpa: self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4248,7 +4262,7 @@ def test_sdpa_can_compile_dynamic(self): self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") for model_class in self.all_model_classes: - if not self._is_composite and not model_class._supports_sdpa: + if not model_class._supports_sdpa: self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4290,7 +4304,7 @@ def test_eager_matches_sdpa_generate(self): self.skipTest(f"{self.__class__.__name__} tests a model that does support generate: skipping this test") for model_class in self.all_generative_model_classes: - if not self._is_composite and not model_class._supports_sdpa: + if not model_class._supports_sdpa: self.skipTest(f"{model_class.__name__} does not support SDPA") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -4454,10 +4468,11 @@ def test_flash_attn_2_generate_use_cache(self): @mark.flash_attn_test def test_flash_attn_2_can_dispatch_composite_models(self): """ - Tests if composite models can dispatch on FA2 if the sub-models supports FA2. + Tests if composite models can dispatch on FA2 if the sub-models support FA2. The tests is needed as we handle differently composite models and we cannot check them with above tests. If any of the sub-models does not support FA2, we'll raise an error when dispatching - that particular sub-model. Otherwise we dispatch safely in all sub-modules. + that particular sub-model. Otherwise we dispatch safely in all sub-models, where "sub-models" are specific + backbone models (LM/vision/audio/etc) """ if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") From d691097d1dec4e47a79d53506f337b38d98c1b47 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 4 Oct 2024 15:35:42 +0200 Subject: [PATCH 59/68] fix copies --- tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py index d2f658f56bd8..23dace68cf21 100644 --- a/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py +++ b/tests/models/recurrent_gemma/test_modeling_recurrent_gemma.py @@ -103,7 +103,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = torch.tril(torch.ones_like(input_ids).to(torch_device)) + input_mask = torch.tril(torch.ones(self.batch_size, self.seq_length)).to(torch_device) token_type_ids = None if self.use_token_type_ids: From 39c032e3fb8507424a21407cfd6e79381d9d2d19 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 4 Oct 2024 15:45:28 +0200 Subject: [PATCH 60/68] style and tests --- examples/modular-transformers/modeling_dummy.py | 4 +--- examples/modular-transformers/modeling_my_new_model2.py | 4 +--- .../models/grounding_dino/modeling_grounding_dino.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/examples/modular-transformers/modeling_dummy.py b/examples/modular-transformers/modeling_dummy.py index c57d41785f6d..51349ecf4ecf 100644 --- a/examples/modular-transformers/modeling_dummy.py +++ b/examples/modular-transformers/modeling_dummy.py @@ -881,9 +881,7 @@ def forward( return_dict = return_dict if return_dict is not None else self.config.use_return_dict if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError( - "You must specify exactly one of input_ids or inputs_embeds" - ) + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") if self.gradient_checkpointing and self.training and use_cache: logger.warning_once( diff --git a/examples/modular-transformers/modeling_my_new_model2.py b/examples/modular-transformers/modeling_my_new_model2.py index a1fabf5d8c51..49cdd2741620 100644 --- a/examples/modular-transformers/modeling_my_new_model2.py +++ b/examples/modular-transformers/modeling_my_new_model2.py @@ -758,9 +758,7 @@ def forward( return_dict = return_dict if return_dict is not None else self.config.use_return_dict if (input_ids is None) ^ (inputs_embeds is not None): - raise ValueError( - "You must specify exactly one of input_ids or inputs_embeds" - ) + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") if self.gradient_checkpointing and self.training and use_cache: logger.warning_once( diff --git a/src/transformers/models/grounding_dino/modeling_grounding_dino.py b/src/transformers/models/grounding_dino/modeling_grounding_dino.py index 3ba3847d3072..aaac7488f430 100644 --- a/src/transformers/models/grounding_dino/modeling_grounding_dino.py +++ b/src/transformers/models/grounding_dino/modeling_grounding_dino.py @@ -2118,7 +2118,7 @@ def __init__(self, config: GroundingDinoConfig): # Create text backbone self.text_backbone = AutoModel.from_config( - config.text_config, add_pooling_layer=False, attn_implementation=config._attn_implementation["text_config"] + config.text_config, add_pooling_layer=False, attn_implementation=config._attn_implementation ) self.text_projection = nn.Linear(config.text_config.hidden_size, config.d_model) From 74b211f879d2c044702b41a4e30b60cb5cacac2d Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 8 Oct 2024 18:02:51 +0200 Subject: [PATCH 61/68] another big update --- src/transformers/configuration_utils.py | 1 + src/transformers/modeling_utils.py | 47 +++++------- .../models/blip_2/modeling_blip_2.py | 32 ++++---- src/transformers/models/clip/modeling_clip.py | 12 +-- .../models/dbrx/configuration_dbrx.py | 74 +++++++++---------- .../modeling_encoder_decoder.py | 29 ++++---- .../models/idefics/modeling_idefics.py | 42 ----------- .../models/idefics2/modeling_idefics2.py | 16 ++-- .../models/idefics3/modeling_idefics3.py | 12 ++- .../instructblip/modeling_instructblip.py | 16 ++-- .../modeling_instructblipvideo.py | 16 ++-- .../models/llava/modeling_llava.py | 10 +-- .../models/llava_next/modeling_llava_next.py | 10 +-- .../modeling_llava_next_video.py | 10 +-- .../modeling_llava_onevision.py | 8 +- .../models/mllama/modeling_mllama.py | 8 +- .../models/musicgen/modeling_musicgen.py | 12 +-- .../modeling_musicgen_melody.py | 12 +-- .../omdet_turbo/modeling_omdet_turbo.py | 4 +- .../models/paligemma/modeling_paligemma.py | 12 ++- .../qwen2_audio/modeling_qwen2_audio.py | 10 +-- .../models/qwen2_vl/modeling_qwen2_vl.py | 4 +- src/transformers/models/rag/modeling_rag.py | 10 +-- .../models/siglip/modeling_siglip.py | 12 +-- .../modeling_speech_encoder_decoder.py | 8 +- .../video_llava/modeling_video_llava.py | 14 ++-- .../models/vipllava/modeling_vipllava.py | 10 +-- .../modeling_vision_encoder_decoder.py | 8 +- .../modeling_vision_text_dual_encoder.py | 10 +-- tests/models/blip_2/test_modeling_blip_2.py | 56 ++++++-------- tests/models/clip/test_modeling_clip.py | 10 +-- .../test_modeling_encoder_decoder.py | 10 +-- .../models/idefics2/test_modeling_idefics2.py | 10 +-- .../test_modeling_instructblip.py | 28 +++---- .../test_modeling_instructblipvideo.py | 28 +++---- .../models/musicgen/test_modeling_musicgen.py | 14 +--- .../test_modeling_musicgen_melody.py | 14 +--- .../qwen2_audio/test_modeling_qwen2_audio.py | 20 ++--- tests/models/siglip/test_modeling_siglip.py | 10 +-- .../test_modeling_speech_encoder_decoder.py | 10 +-- .../test_modeling_vision_encoder_decoder.py | 10 +-- tests/test_modeling_common.py | 16 ++-- 42 files changed, 254 insertions(+), 451 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index a5a17260d5e4..b3306b2863e1 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -293,6 +293,7 @@ def __init__(self, **kwargs): # Attention implementation to use, if relevant. self._attn_implementation_internal = kwargs.pop("attn_implementation", None) + self._attn_implementation_autoset = False # Drop the transformers version info self.transformers_version = kwargs.pop("transformers_version", None) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 9c4c666e17bf..3a42b6c8f645 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1419,9 +1419,10 @@ def __init__(self, config: PretrainedConfig, *inputs, **kwargs): f"`model = {self.__class__.__name__}.from_pretrained(PRETRAINED_MODEL_NAME)`" ) # Save config and origin of the pretrained weights if given in model - config = self._autoset_attn_implementation( - config, torch_dtype=torch.get_default_dtype(), check_device_map=False - ) + if not config._attn_implementation_autoset: + config = self._autoset_attn_implementation( + config, torch_dtype=torch.get_default_dtype(), check_device_map=False + ) self.config = config self.name_or_path = config.name_or_path @@ -1517,12 +1518,13 @@ def _from_config(cls, config, **kwargs): attn_implementation = None config._attn_implementation = kwargs.pop("attn_implementation", attn_implementation) - config = cls._autoset_attn_implementation( - config, - use_flash_attention_2=use_flash_attention_2, - check_device_map=False, - torch_dtype=torch_dtype, - ) + if not config._attn_implementation_autoset: + config = cls._autoset_attn_implementation( + config, + use_flash_attention_2=use_flash_attention_2, + check_device_map=False, + torch_dtype=torch_dtype, + ) if is_deepspeed_zero3_enabled(): import deepspeed @@ -1588,27 +1590,17 @@ def _autoset_attn_implementation( # where keys are sub-config names. But most people will specify one `str` which means that should dispatch it # for all sub-models. # Below we check if a config is composite and manually prepare a dict of attn impl if not already passed as a dict. - # Later each sub-module will dispatch with its own attn impl, by calling `_from_config(attn_impl="sdpa/FA2/eager")` + # Later each sub-module will dispatch with its own attn impl, by calling `XXXModel._from_config(config.text_config)` # If any of sub-modules doesm't support requested attn, an error will be raised. See https://github.com/huggingface/transformers/pull/32238 - sub_configs = { - key: getattr(config, key) for key in config if isinstance(getattr(config, key), PretrainedConfig) - } - if sub_configs: - attn_implementation_per_subconfig = {} - for key, sub_config in sub_configs.items(): - attn_implementation_per_subconfig[key] = ( + for key in config: + if isinstance(getattr(config, key), PretrainedConfig): + sub_config = getattr(config, key) + curr_attn_implementation = ( requested_attn_implementation if not isinstance(requested_attn_implementation, dict) - else requested_attn_implementation.get(key) + else requested_attn_implementation.get(key, None) ) - - # Some models have nested configs where text config holds vision config - # inside itself. So we don't set their attn implementation as dicts and leave - # everything as it was. There are only 3 models like that (Qwen2_VL, GIT, Chameleon) - # and all of them support all attn implementations - if len(attn_implementation_per_subconfig.keys()) != 1: - config._attn_implementation = attn_implementation_per_subconfig - requested_attn_implementation = config._attn_implementation + sub_config._attn_implementation_internal = curr_attn_implementation if use_flash_attention_2: logger.warning_once( @@ -1641,10 +1633,11 @@ def _autoset_attn_implementation( ) torch.backends.cuda.enable_flash_sdp(False) elif isinstance(requested_attn_implementation, dict): - config._attn_implementation = requested_attn_implementation + config._attn_implementation = None else: config._attn_implementation = "eager" + config._attn_implementation_autoset = True return config @classmethod diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 496695da064d..bb3370033972 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -410,6 +410,8 @@ class Blip2PreTrainedModel(PreTrainedModel): config_class = Blip2Config base_model_prefix = "blip" supports_gradient_checkpointing = True + _supports_flash_attn_2 = False + _supports_sdpa = False _no_split_modules = [ "Blip2Attention", @@ -1446,25 +1448,22 @@ class Blip2Model(Blip2PreTrainedModel): config_class = Blip2Config main_input_name = "pixel_values" + _supports_flash_attn_2 = True + _supports_sdpa = True + def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_model = Blip2VisionModel._from_config(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = Blip2QFormerModel(config.qformer_config) self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: - language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForCausalLM.from_config(config.text_config) else: - language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForSeq2SeqLM.from_config(config.text_config) # Update _tied_weights_keys using the base model used. if language_model._tied_weights_keys is not None: @@ -2013,25 +2012,22 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin): config_class = Blip2Config main_input_name = "pixel_values" + _supports_flash_attn_2 = True + _supports_sdpa = True + def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_model = Blip2VisionModel._from_config(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = Blip2QFormerModel(config.qformer_config) self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: - language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForCausalLM.from_config(config.text_config) else: - language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForSeq2SeqLM.from_config(config.text_config) # Update _tied_weights_keys using the base model used. if language_model._tied_weights_keys is not None: diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py index 6562683f38aa..eb8f927e7b5e 100644 --- a/src/transformers/models/clip/modeling_clip.py +++ b/src/transformers/models/clip/modeling_clip.py @@ -1204,14 +1204,10 @@ def __init__(self, config: CLIPConfig): self.text_embed_dim = text_config.hidden_size self.vision_embed_dim = vision_config.hidden_size - text_model = CLIPTextModel._from_config( - text_config, attn_implementation=config._attn_implementation["text_config"] - ) + text_model = CLIPTextModel._from_config(text_config) self.text_model = text_model.text_model - vision_model = CLIPVisionModel._from_config( - vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + vision_model = CLIPVisionModel._from_config(vision_config) self.vision_model = vision_model.vision_model self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False) @@ -1594,9 +1590,7 @@ def __init__(self, config: CLIPConfig) -> None: super().__init__(config) self.num_labels = config.num_labels - vision_model = CLIPVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + vision_model = CLIPVisionModel._from_config(config.vision_config) self.vision_model = vision_model.vision_model # Classifier head diff --git a/src/transformers/models/dbrx/configuration_dbrx.py b/src/transformers/models/dbrx/configuration_dbrx.py index 1052ebbe8991..dde5232ae5cc 100644 --- a/src/transformers/models/dbrx/configuration_dbrx.py +++ b/src/transformers/models/dbrx/configuration_dbrx.py @@ -14,8 +14,7 @@ # limitations under the License. """DBRX model configuration""" -import copy -from typing import Any, Dict, Optional +from typing import Any, Optional from ...configuration_utils import PretrainedConfig from ...utils import logging @@ -24,7 +23,7 @@ logger = logging.get_logger(__name__) -class DbrxAttentionConfig: +class DbrxAttentionConfig(PretrainedConfig): """Configuration class for Dbrx Attention. [`DbrxAttention`] class. It is used to instantiate attention layers @@ -50,6 +49,7 @@ def __init__( rope_theta: float = 10000.0, **kwargs: Any, ): + super().__init__(**kwargs) self.attn_pdrop = attn_pdrop self.clip_qkv = clip_qkv self.kv_n_heads = kv_n_heads @@ -61,20 +61,25 @@ def __init__( if len(kwargs) != 0: raise ValueError(f"Found unknown {kwargs=}") - def to_dict(self) -> Dict[str, Any]: - """ - Serializes this instance to a Python dictionary. + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) - Returns: - `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. - """ - output = copy.deepcopy(self.__dict__) - if hasattr(self.__class__, "model_type"): - output["model_type"] = self.__class__.model_type - return output + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["attn_config"] -class DbrxFFNConfig: + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxFFNConfig(PretrainedConfig): """Configuration class for Dbrx FFN. [`DbrxFFN`] class. It is used to instantiate feedforward layers according to @@ -123,17 +128,22 @@ def __init__( if len(kwargs) != 0: raise ValueError(f"Found unknown {kwargs=}") - def to_dict(self) -> Dict[str, Any]: - """ - Serializes this instance to a Python dictionary. + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["ffn_config"] - Returns: - `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. - """ - output = copy.deepcopy(self.__dict__) - if hasattr(self.__class__, "model_type"): - output["model_type"] = self.__class__.model_type - return output + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) class DbrxConfig(PretrainedConfig): @@ -246,19 +256,3 @@ def __init__( raise ValueError("tie_word_embeddings is not supported for DBRX models.") super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) - - def to_dict(self) -> Dict[str, Any]: - """ - Serializes this instance to a Python dictionary. - - Returns: - `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. - """ - output = super().to_dict() - - for key, value in output.items(): - if key in ["ffn_config", "attn_config"]: - value = value.to_dict() - output[key] = value - - return output diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index c56110967e4b..304f02750b64 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -179,6 +179,8 @@ class EncoderDecoderModel(PreTrainedModel): main_input_name = "input_ids" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False + _supports_flash_attn_2 = True + _supports_sdpa = True def __init__( self, @@ -209,31 +211,30 @@ def __init__( if encoder is None: from ..auto.modeling_auto import AutoModel - encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation["encoder"]) + encoder = AutoModel.from_config(config.encoder) if decoder is None: from ..auto.modeling_auto import AutoModelForCausalLM - decoder = AutoModelForCausalLM.from_config( - config.decoder, attn_implementation=config._attn_implementation["decoder"] - ) + decoder = AutoModelForCausalLM.from_config(config.decoder) self.encoder = encoder self.decoder = decoder - if self.encoder.config.to_dict() != self.config.encoder.to_dict(): - logger.warning( - f"Config of the encoder: {self.encoder.__class__} is overwritten by shared encoder config:" - f" {self.config.encoder}" - ) - if self.decoder.config.to_dict() != self.config.decoder.to_dict(): - logger.warning( - f"Config of the decoder: {self.decoder.__class__} is overwritten by shared decoder config:" - f" {self.config.decoder}" - ) + # if self.encoder.config.to_dict() != self.config.encoder.to_dict(): + # logger.warning( + # f"Config of the encoder: {self.encoder.__class__} is overwritten by shared encoder config:" + # f" {self.config.encoder}" + # ) + # if self.decoder.config.to_dict() != self.config.decoder.to_dict(): + # logger.warning( + # f"Config of the decoder: {self.decoder.__class__} is overwritten by shared decoder config:" + # f" {self.config.decoder}" + # ) # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced + # update `_attn_implementation` because the attn is set a a deepcopied config within PreTrainedMolde self.config.encoder._attn_implementation = self.encoder.config._attn_implementation self.config.decoder._attn_implementation = self.decoder.config._attn_implementation self.encoder.config = self.config.encoder diff --git a/src/transformers/models/idefics/modeling_idefics.py b/src/transformers/models/idefics/modeling_idefics.py index 31d3476f2a48..985a0dd40449 100644 --- a/src/transformers/models/idefics/modeling_idefics.py +++ b/src/transformers/models/idefics/modeling_idefics.py @@ -980,48 +980,6 @@ def _init_weights(self, module): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - # Adapted from transformers.modeling_utils.PreTrainedModel._autoset_attn_implementation - @classmethod - def _autoset_attn_implementation( - cls, - config, - use_flash_attention_2: bool = False, - torch_dtype: Optional[torch.dtype] = None, - device_map: Optional[Union[str, Dict[str, int]]] = None, - check_device_map: bool = True, - ): - requested_attn_implementation = None - if hasattr(config, "_attn_implementation_internal") and config._attn_implementation_internal is not None: - if config._attn_implementation != "flash_attention_2" and use_flash_attention_2: - raise ValueError( - f'Both attn_implementation="{config._attn_implementation}" and `use_flash_attention_2=True` were used when loading the model, which are not compatible.' - ' We recommend to just use `attn_implementation="flash_attention_2"` when loading the model.' - ) - - if not isinstance(config._attn_implementation, dict) and config._attn_implementation not in [ - "eager", - "sdpa", - "flash_attention_2", - ]: - message = f'Specified `attn_implementation="{config._attn_implementation}"` is not supported. The only possible arguments are `attn_implementation="eager"` (manual attention implementation)' - if cls._supports_flash_attn_2: - message += ', `"attn_implementation=flash_attention_2"` (implementation using flash attention 2)' - if cls._supports_sdpa: - message += ', `"attn_implementation=sdpa"` (implementation using torch.nn.functional.scaled_dot_product_attention)' - raise ValueError(message + ".") - - # If a config is passed with a preset attn_implementation, we skip the automatic dispatch and use the user-provided config, with hard checks that the requested attention implementation is available. - requested_attn_implementation = config._attn_implementation_internal - - # IDEFICS has ahrd requirement on SDPA - if requested_attn_implementation not in ["sdpa", None]: - logger.warning_once( - f"Idefics supports only SDPA attention, but the model being loaded with {requested_attn_implementation} " - "Falling back to SDPA. If you need other attention implementations, please open an issue." - ) - config._attn_implementation = "sdpa" - return config - LLAMA_INPUTS_DOCSTRING = r""" Args: diff --git a/src/transformers/models/idefics2/modeling_idefics2.py b/src/transformers/models/idefics2/modeling_idefics2.py index 8a59905d4381..206317582eab 100644 --- a/src/transformers/models/idefics2/modeling_idefics2.py +++ b/src/transformers/models/idefics2/modeling_idefics2.py @@ -600,6 +600,7 @@ class Idefics2PreTrainedModel(PreTrainedModel): _no_split_modules = ["Idefics2VisionAttention", "Idefics2MLP", "Idefics2PerceiverLayer", "Idefics2DecoderLayer"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _supports_sdpa = True _supports_cache_class = True def _init_weights(self, module): @@ -647,6 +648,7 @@ def _init_weights(self, module): ) class Idefics2VisionTransformer(Idefics2PreTrainedModel): _supports_sdpa = False + config_class = Idefics2VisionConfig def __init__(self, config: Idefics2VisionConfig): super().__init__(config) @@ -1149,9 +1151,7 @@ def __init__(self, config): output_size=config.text_config.hidden_size, hidden_act=config.text_config.hidden_act, ) - self.perceiver_resampler = Idefics2PerceiverResampler._from_config( - config.perceiver_config, attn_implementation=config._attn_implementation["perceiver_config"] - ) + self.perceiver_resampler = Idefics2PerceiverResampler._from_config(config.perceiver_config) def forward(self, image_hidden_states, attention_mask): image_hidden_states = self.modality_projection(image_hidden_states) @@ -1239,18 +1239,14 @@ def __init__(self, config: Idefics2Config): self.padding_idx = self.config.text_config.pad_token_id self.vocab_size = self.config.text_config.vocab_size - self.vision_model = Idefics2VisionTransformer._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_model = Idefics2VisionTransformer._from_config(config.vision_config) self.connector = Idefics2Connector(config) - self.text_model = AutoModel.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.text_model = AutoModel.from_config(config.text_config) self.image_seq_len = config.perceiver_config.resampler_n_latents self.image_token_id = self.config.image_token_id - self._use_flash_attention_2 = config._attn_implementation["text_config"] == "flash_attention_2" + self._use_flash_attention_2 = config.text_config._attn_implementation == "flash_attention_2" self.post_init() diff --git a/src/transformers/models/idefics3/modeling_idefics3.py b/src/transformers/models/idefics3/modeling_idefics3.py index 196fe30144d5..e5852fd5d1f6 100644 --- a/src/transformers/models/idefics3/modeling_idefics3.py +++ b/src/transformers/models/idefics3/modeling_idefics3.py @@ -620,6 +620,7 @@ class Idefics3PreTrainedModel(PreTrainedModel): _no_split_modules = ["Idefics3VisionAttention", "Idefics3DecoderLayer"] _skip_keys_device_placement = "past_key_values" _supports_flash_attn_2 = True + _supports_sdpa = True _supports_cache_class = True # Copied from transformers.models.idefics2.modeling_idefics2.Idefics2PreTrainedModel._init_weights @@ -666,6 +667,7 @@ def _init_weights(self, module): ) class Idefics3VisionTransformer(Idefics3PreTrainedModel): config_class = Idefics3VisionConfig + _supports_sdpa = False def __init__(self, config: Idefics3VisionConfig): super().__init__(config) @@ -823,20 +825,16 @@ def __init__(self, config: Idefics3Config): self.padding_idx = self.config.text_config.pad_token_id self.vocab_size = self.config.text_config.vocab_size - self.vision_model = Idefics3VisionTransformer._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_model = Idefics3VisionTransformer._from_config(config.vision_config) self.connector = Idefics3Connector(config) - self.text_model = AutoModel.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.text_model = AutoModel.from_config(config.text_config) self.image_seq_len = int( ((config.vision_config.image_size // config.vision_config.patch_size) ** 2) / (config.scale_factor**2) ) self.image_token_id = self.config.image_token_id - self._use_flash_attention_2 = "flash_attention_2" in config._attn_implementation.values() + self._use_flash_attention_2 = config.text_config._attn_implementation == "flash_attention_2" self.post_init() diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index a89ec5254503..73b175a7c10f 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -315,6 +315,8 @@ class InstructBlipPreTrainedModel(PreTrainedModel): config_class = InstructBlipConfig base_model_prefix = "blip" supports_gradient_checkpointing = True + _supports_flash_attn_2 = False + _supports_sdpa = False _no_split_modules = [ "InstructBlipQFormerEmbeddings", @@ -1287,13 +1289,13 @@ def forward( class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, GenerationMixin): config_class = InstructBlipConfig main_input_name = "pixel_values" + _supports_flash_attn_2 = True + _supports_sdpa = True def __init__(self, config: InstructBlipConfig): super().__init__(config) - self.vision_model = InstructBlipVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_model = InstructBlipVisionModel._from_config(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = InstructBlipQFormerModel(config.qformer_config) @@ -1301,13 +1303,9 @@ def __init__(self, config: InstructBlipConfig): self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: - language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForCausalLM.from_config(config.text_config) else: - language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForSeq2SeqLM.from_config(config.text_config) if language_model._no_split_modules is not None: self._no_split_modules.extend(language_model._no_split_modules) diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index ce536b1e5bc7..250a0a41b442 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -322,6 +322,8 @@ class InstructBlipVideoPreTrainedModel(PreTrainedModel): config_class = InstructBlipVideoConfig base_model_prefix = "blip" supports_gradient_checkpointing = True + _supports_flash_attn_2 = False + _supports_sdpa = False _no_split_modules = [ "InstructBlipVideoQFormerEmbeddings", @@ -1294,13 +1296,13 @@ def forward( class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel, GenerationMixin): config_class = InstructBlipVideoConfig main_input_name = "pixel_values" + _supports_flash_attn_2 = False + _supports_sdpa = False def __init__(self, config: InstructBlipVideoConfig): super().__init__(config) - self.vision_model = InstructBlipVideoVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_model = InstructBlipVideoVisionModel._from_config(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = InstructBlipVideoQFormerModel(config.qformer_config) @@ -1308,13 +1310,9 @@ def __init__(self, config: InstructBlipVideoConfig): self.language_projection = nn.Linear(config.qformer_config.hidden_size, config.text_config.hidden_size) if config.use_decoder_only_language_model: - language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForCausalLM.from_config(config.text_config) else: - language_model = AutoModelForSeq2SeqLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForSeq2SeqLM.from_config(config.text_config) if language_model._no_split_modules is not None: self._no_split_modules.extend(language_model._no_split_modules) diff --git a/src/transformers/models/llava/modeling_llava.py b/src/transformers/models/llava/modeling_llava.py index e5dfdd0c022d..10282050aec0 100644 --- a/src/transformers/models/llava/modeling_llava.py +++ b/src/transformers/models/llava/modeling_llava.py @@ -126,6 +126,8 @@ class LlavaPreTrainedModel(PreTrainedModel): _no_split_modules = ["LlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True + _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): # important: this ported version of Llava isn't meant for training from scratch - only @@ -232,15 +234,11 @@ def _init_weights(self, module): class LlavaForConditionalGeneration(LlavaPreTrainedModel, GenerationMixin): def __init__(self, config: LlavaConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_tower = AutoModel.from_config(config.vision_config) self.multi_modal_projector = LlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size - self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.language_model = AutoModelForCausalLM.from_config(config.text_config) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/llava_next/modeling_llava_next.py b/src/transformers/models/llava_next/modeling_llava_next.py index 2c2d50374358..0ab1e3490ef0 100644 --- a/src/transformers/models/llava_next/modeling_llava_next.py +++ b/src/transformers/models/llava_next/modeling_llava_next.py @@ -235,6 +235,8 @@ class LlavaNextPreTrainedModel(PreTrainedModel): _no_split_modules = ["LlavaNextVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True + _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): # important: this ported version of LlavaNext isn't meant for training from scratch - only @@ -344,18 +346,14 @@ def _init_weights(self, module): class LlavaNextForConditionalGeneration(LlavaNextPreTrainedModel, GenerationMixin): def __init__(self, config: LlavaNextConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_tower = AutoModel.from_config(config.vision_config) self.multi_modal_projector = LlavaNextMultiModalProjector(config) embed_std = 1 / math.sqrt(config.text_config.hidden_size) self.image_newline = nn.Parameter(torch.randn(config.text_config.hidden_size, dtype=self.dtype) * embed_std) self.vocab_size = config.text_config.vocab_size - self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.language_model = AutoModelForCausalLM.from_config(config.text_config) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides self.post_init() diff --git a/src/transformers/models/llava_next_video/modeling_llava_next_video.py b/src/transformers/models/llava_next_video/modeling_llava_next_video.py index 120536a64f7e..f6c37e0da1a6 100644 --- a/src/transformers/models/llava_next_video/modeling_llava_next_video.py +++ b/src/transformers/models/llava_next_video/modeling_llava_next_video.py @@ -280,6 +280,8 @@ class LlavaNextVideoPreTrainedModel(PreTrainedModel): _no_split_modules = ["LlavaNextVideoVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True + _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): # important: this ported version of LlavaNextVideo isn't meant for training from scratch - only @@ -392,18 +394,14 @@ def __init__( config: LlavaNextVideoConfig, ): super().__init__(config) - self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_tower = AutoModel.from_config(config.vision_config) self.multi_modal_projector = LlavaNextVideoMultiModalProjector(config) embed_std = 1 / math.sqrt(config.text_config.hidden_size) self.image_newline = nn.Parameter(torch.randn(config.text_config.hidden_size, dtype=self.dtype) * embed_std) self.vocab_size = config.text_config.vocab_size - self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.language_model = AutoModelForCausalLM.from_config(config.text_config) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides self.vision_resampler = LlavaNextVideoPooler(config) diff --git a/src/transformers/models/llava_onevision/modeling_llava_onevision.py b/src/transformers/models/llava_onevision/modeling_llava_onevision.py index 6f9302ff47d0..0a410a41a4dc 100644 --- a/src/transformers/models/llava_onevision/modeling_llava_onevision.py +++ b/src/transformers/models/llava_onevision/modeling_llava_onevision.py @@ -363,18 +363,14 @@ def _init_weights(self, module): class LlavaOnevisionForConditionalGeneration(LlavaOnevisionPreTrainedModel, GenerationMixin): def __init__(self, config: LlavaOnevisionConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_tower = AutoModel.from_config(config.vision_config) self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config) embed_std = 1 / math.sqrt(config.text_config.hidden_size) self.image_newline = nn.Parameter(torch.randn(config.text_config.hidden_size, dtype=self.dtype) * embed_std) self.vocab_size = config.text_config.vocab_size - self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.language_model = AutoModelForCausalLM.from_config(config.text_config) self.post_init() # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.get_input_embeddings diff --git a/src/transformers/models/mllama/modeling_mllama.py b/src/transformers/models/mllama/modeling_mllama.py index 12118c4270e7..ab09b889046c 100644 --- a/src/transformers/models/mllama/modeling_mllama.py +++ b/src/transformers/models/mllama/modeling_mllama.py @@ -2038,12 +2038,8 @@ def __init__(self, config: MllamaConfig): self.vision_output_dim = config.vision_config.vision_output_dim self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 - self.vision_model = MllamaVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) - self.language_model = MllamaForCausalLM._from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.vision_model = MllamaVisionModel._from_config(config.vision_config) + self.language_model = MllamaForCausalLM._from_config(config.text_config) self.multi_modal_projector = nn.Linear( config.vision_config.vision_output_dim, config.text_config.hidden_size, diff --git a/src/transformers/models/musicgen/modeling_musicgen.py b/src/transformers/models/musicgen/modeling_musicgen.py index 536e9f0dcf22..c829bf6f0722 100644 --- a/src/transformers/models/musicgen/modeling_musicgen.py +++ b/src/transformers/models/musicgen/modeling_musicgen.py @@ -1703,21 +1703,15 @@ def __init__( if text_encoder is None: from ..auto.modeling_auto import AutoModelForTextEncoding - text_encoder = AutoModelForTextEncoding.from_config( - config.text_encoder, attn_implementation=config._attn_implementation["text_encoder"] - ) + text_encoder = AutoModelForTextEncoding.from_config(config.text_encoder) if audio_encoder is None: from ..auto.modeling_auto import AutoModel - audio_encoder = AutoModel.from_config( - config.audio_encoder, attn_implementation=config._attn_implementation["audio_encoder"] - ) + audio_encoder = AutoModel.from_config(config.audio_encoder) if decoder is None: - decoder = MusicgenForCausalLM._from_config( - config.decoder, attn_implementation=config._attn_implementation["decoder"] - ) + decoder = MusicgenForCausalLM._from_config(config.decoder) self.text_encoder = text_encoder self.audio_encoder = audio_encoder diff --git a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py index c6ae54aab251..8348ddde9337 100644 --- a/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py +++ b/src/transformers/models/musicgen_melody/modeling_musicgen_melody.py @@ -1620,19 +1620,13 @@ def __init__( super().__init__(config) if text_encoder is None: - text_encoder = AutoModelForTextEncoding.from_config( - config.text_encoder, attn_implementation=config._attn_implementation["text_encoder"] - ) + text_encoder = AutoModelForTextEncoding.from_config(config.text_encoder) if audio_encoder is None: - audio_encoder = AutoModel.from_config( - config.audio_encoder, attn_implementation=config._attn_implementation["audio_encoder"] - ) + audio_encoder = AutoModel.from_config(config.audio_encoder) if decoder is None: - decoder = MusicgenMelodyForCausalLM._from_config( - config.decoder, attn_implementation=config._attn_implementation["decoder"] - ) + decoder = MusicgenMelodyForCausalLM._from_config(config.decoder) self.text_encoder = text_encoder self.audio_encoder = audio_encoder diff --git a/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py b/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py index a8859eb69acf..0f44e4bd4020 100644 --- a/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py +++ b/src/transformers/models/omdet_turbo/modeling_omdet_turbo.py @@ -288,9 +288,7 @@ def put(self, key, value) -> None: class OmDetTurboLanguageBackbone(nn.Module): def __init__(self, config: OmDetTurboConfig): super().__init__() - self.model = AutoModel.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.model = AutoModel.from_config(config.text_config) self.text_projection = nn.Parameter(torch.zeros(config.text_projection_in_dim, config.text_projection_out_dim)) def forward(self, hidden_states, mask=None, encode_type="task"): diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index a4b96809fba5..f666b432d778 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -195,6 +195,9 @@ class PaliGemmaPreTrainedModel(PreTrainedModel): _supports_cache_class = True _supports_quantized_cache = True _supports_static_cache = True + _supports_cache_class = True + _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): # important: this ported version of PaliGemmaisn't meant for training from scratch - only @@ -295,16 +298,11 @@ def _init_weights(self, module): class PaliGemmaForConditionalGeneration(PaliGemmaPreTrainedModel, GenerationMixin): def __init__(self, config: PaliGemmaConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config( - config=config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_tower = AutoModel.from_config(config=config.vision_config) self.multi_modal_projector = PaliGemmaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size - self._attn_implementation = config._attn_implementation - language_model = AutoModelForCausalLM.from_config( - config=config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + language_model = AutoModelForCausalLM.from_config(config=config.text_config) if language_model._tied_weights_keys is not None: self._tied_weights_keys = [f"language_model.{k}" for k in language_model._tied_weights_keys] diff --git a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py index 81c1a42e2170..e31494acc3bb 100644 --- a/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py +++ b/src/transformers/models/qwen2_audio/modeling_qwen2_audio.py @@ -543,6 +543,8 @@ class Qwen2AudioPreTrainedModel(PreTrainedModel): supports_gradient_checkpointing = True _no_split_modules = ["Qwen2AudioAttention"] _skip_keys_device_placement = "past_key_values" + _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): # important: this ported version of Qwen2Audio isn't meant for training from scratch - only @@ -850,15 +852,11 @@ def forward(self, audio_features): class Qwen2AudioForConditionalGeneration(Qwen2AudioPreTrainedModel, GenerationMixin): def __init__(self, config: Qwen2AudioConfig): super().__init__(config) - self.audio_tower = AutoModel.from_config( - config.audio_config, attn_implementation=config._attn_implementation["audio_config"] - ) + self.audio_tower = AutoModel.from_config(config.audio_config) self.multi_modal_projector = Qwen2AudioMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size - self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.language_model = AutoModelForCausalLM.from_config(config.text_config) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self._padding_side = "left" # set it to left by default, user can use setter to change padding_sides self.post_init() diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 9ca33395e923..9ad619f2b333 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -1421,9 +1421,7 @@ class Qwen2VLForConditionalGeneration(Qwen2VLPreTrainedModel, GenerationMixin): def __init__(self, config): super().__init__(config) - self.visual = Qwen2VisionTransformerPretrainedModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation - ) + self.visual = Qwen2VisionTransformerPretrainedModel._from_config(config.vision_config) self.model = Qwen2VLModel(config) self.vocab_size = config.vocab_size self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py index 6ca6b8901f07..4470a2220e21 100644 --- a/src/transformers/models/rag/modeling_rag.py +++ b/src/transformers/models/rag/modeling_rag.py @@ -232,6 +232,8 @@ class RagPreTrainedModel(PreTrainedModel): config_class = RagConfig base_model_prefix = "rag" + _supports_flash_attn_2 = True + _supports_sdpa = True @classmethod def from_pretrained(cls, *args, **kwargs): @@ -506,16 +508,12 @@ def __init__( if question_encoder is None: from ..auto.modeling_auto import AutoModel - question_encoder = AutoModel.from_config( - config.question_encoder, attn_implementation=config._attn_implementation["question_encoder"] - ) + question_encoder = AutoModel.from_config(config.question_encoder) if generator is None: from ..auto.modeling_auto import AutoModelForSeq2SeqLM - generator = AutoModelForSeq2SeqLM.from_config( - config.generator, attn_implementation=config._attn_implementation["generator"] - ) + generator = AutoModelForSeq2SeqLM.from_config(config.generator) self.retriever = retriever if self.retriever is not None: diff --git a/src/transformers/models/siglip/modeling_siglip.py b/src/transformers/models/siglip/modeling_siglip.py index f2d1d10e3773..77ec945e98fb 100644 --- a/src/transformers/models/siglip/modeling_siglip.py +++ b/src/transformers/models/siglip/modeling_siglip.py @@ -1219,12 +1219,8 @@ def __init__(self, config: SiglipConfig): vision_config = config.vision_config # First, initialize the text and vision models with proper attention implementation - text_model = SiglipTextModel._from_config( - text_config, attn_implementation=config._attn_implementation["text_config"] - ) - vision_model = SiglipVisionModel._from_config( - vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + text_model = SiglipTextModel._from_config(text_config) + vision_model = SiglipVisionModel._from_config(vision_config) # Second, get the text and vision submodules (for backward compatibility) self.text_model = text_model.text_model @@ -1459,9 +1455,7 @@ def __init__(self, config: SiglipConfig) -> None: # Create the vision model with proper attention # and take only vision_model submodule (for backward compatibility) - vision_model = SiglipVisionModel._from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + vision_model = SiglipVisionModel._from_config(config.vision_config) self.vision_model = vision_model.vision_model # Classifier head diff --git a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py index 012ab8dfd84d..bfdded5d8368 100644 --- a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py +++ b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py @@ -182,6 +182,8 @@ class SpeechEncoderDecoderModel(PreTrainedModel): main_input_name = "inputs" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False + _supports_flash_attn_2 = True + _supports_sdpa = True def __init__( self, @@ -212,12 +214,10 @@ def __init__( super().__init__(config) if encoder is None: - encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation["encoder"]) + encoder = AutoModel.from_config(config.encoder) if decoder is None: - decoder = AutoModelForCausalLM.from_config( - config.decoder, attn_implementation=config._attn_implementation["decoder"] - ) + decoder = AutoModelForCausalLM.from_config(config.decoder) self.encoder = encoder self.decoder = decoder diff --git a/src/transformers/models/video_llava/modeling_video_llava.py b/src/transformers/models/video_llava/modeling_video_llava.py index a68018b80282..822e2374f3a3 100644 --- a/src/transformers/models/video_llava/modeling_video_llava.py +++ b/src/transformers/models/video_llava/modeling_video_llava.py @@ -127,6 +127,8 @@ class VideoLlavaPreTrainedModel(PreTrainedModel): _no_split_modules = ["VideoLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True + _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): std = ( @@ -234,18 +236,12 @@ def _init_weights(self, module): class VideoLlavaForConditionalGeneration(VideoLlavaPreTrainedModel, GenerationMixin): def __init__(self, config: VideoLlavaConfig): super().__init__(config) - self.video_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) - self.image_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.video_tower = AutoModel.from_config(config.vision_config) + self.image_tower = AutoModel.from_config(config.vision_config) self.multi_modal_projector = VideoLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size - self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.language_model = AutoModelForCausalLM.from_config(config.text_config) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/vipllava/modeling_vipllava.py b/src/transformers/models/vipllava/modeling_vipllava.py index e49765da55ef..30215fbc374f 100644 --- a/src/transformers/models/vipllava/modeling_vipllava.py +++ b/src/transformers/models/vipllava/modeling_vipllava.py @@ -133,6 +133,8 @@ class VipLlavaPreTrainedModel(PreTrainedModel): _no_split_modules = ["VipLlavaVisionAttention"] _skip_keys_device_placement = "past_key_values" _supports_cache_class = True + _supports_flash_attn_2 = True + _supports_sdpa = True def _init_weights(self, module): # important: this ported version of VipLlava isn't meant for training from scratch - only @@ -235,15 +237,11 @@ def _init_weights(self, module): class VipLlavaForConditionalGeneration(VipLlavaPreTrainedModel, GenerationMixin): def __init__(self, config: VipLlavaConfig): super().__init__(config) - self.vision_tower = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + self.vision_tower = AutoModel.from_config(config.vision_config) self.multi_modal_projector = VipLlavaMultiModalProjector(config) self.vocab_size = config.text_config.vocab_size - self.language_model = AutoModelForCausalLM.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + self.language_model = AutoModelForCausalLM.from_config(config.text_config) self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1 self.post_init() diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py index 3908b315e01e..d676bab2399b 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py @@ -160,6 +160,8 @@ class VisionEncoderDecoderModel(PreTrainedModel): main_input_name = "pixel_values" supports_gradient_checkpointing = True _supports_param_buffer_assignment = False + _supports_flash_attn_2 = True + _supports_sdpa = True def __init__( self, @@ -190,12 +192,10 @@ def __init__( super().__init__(config) if encoder is None: - encoder = AutoModel.from_config(config.encoder, attn_implementation=config._attn_implementation["encoder"]) + encoder = AutoModel.from_config(config.encoder) if decoder is None: - decoder = AutoModelForCausalLM.from_config( - config.decoder, attn_implementation=config._attn_implementation["decoder"] - ) + decoder = AutoModelForCausalLM.from_config(config.decoder) self.encoder = encoder self.decoder = decoder diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py index 653bf8963a0c..4b39de3df1c8 100755 --- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py +++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py @@ -161,6 +161,8 @@ def clip_loss(similarity: torch.Tensor) -> torch.Tensor: class VisionTextDualEncoderModel(PreTrainedModel): config_class = VisionTextDualEncoderConfig base_model_prefix = "vision_text_dual_encoder" + _supports_flash_attn_2 = True + _supports_sdpa = True def __init__( self, @@ -184,14 +186,10 @@ def __init__( if isinstance(config.vision_config, CLIPVisionConfig): vision_model = CLIPVisionModel(config.vision_config) else: - vision_model = AutoModel.from_config( - config.vision_config, attn_implementation=config._attn_implementation["vision_config"] - ) + vision_model = AutoModel.from_config(config.vision_config) if text_model is None: - text_model = AutoModel.from_config( - config.text_config, attn_implementation=config._attn_implementation["text_config"] - ) + text_model = AutoModel.from_config(config.text_config) self.vision_model = vision_model self.text_model = text_model diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 80e8a3ff0e11..27b616767cbe 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -517,29 +517,23 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) - text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" - vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" - qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" + vision_attn = "sdpa" if model.vision_model._supports_sdpa else "eager" + qformer_attn = "sdpa" if model.qformer._supports_sdpa else "eager" # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue( - model_sdpa.config._attn_implementation - == {"text_config": None, "vision_config": None, "qformer_config": None} - ) - self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) - self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) - self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model.language_model.config._attn_implementation == text_attn) + self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) - self.assertTrue( - model_eager.config._attn_implementation - == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} - ) - self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.qformer.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ @@ -858,29 +852,23 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) - text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" - vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" - qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" + vision_attn = "sdpa" if model.vision_model._supports_sdpa else "eager" + qformer_attn = "sdpa" if model.qformer._supports_sdpa else "eager" # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue( - model_sdpa.config._attn_implementation - == {"text_config": None, "vision_config": None, "qformer_config": None} - ) - self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) - self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) - self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model.language_model.config._attn_implementation == text_attn) + self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) - self.assertTrue( - model_eager.config._attn_implementation - == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} - ) - self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.qformer.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/clip/test_modeling_clip.py b/tests/models/clip/test_modeling_clip.py index b8ce4163b8d5..a7c8c8ef8410 100644 --- a/tests/models/clip/test_modeling_clip.py +++ b/tests/models/clip/test_modeling_clip.py @@ -221,14 +221,8 @@ def test_sdpa_can_dispatch_composite_models(self): self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") - if hasattr(model_sdpa.config, "text_config"): - self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} - ) - else: - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_eager.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py index 1683e4773553..0ee4b75ed803 100644 --- a/tests/models/encoder_decoder/test_modeling_encoder_decoder.py +++ b/tests/models/encoder_decoder/test_modeling_encoder_decoder.py @@ -697,11 +697,10 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 - # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - self.assertTrue(model_sdpa.config._attn_implementation == {"encoder": None, "decoder": None}) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.encoder.config._attn_implementation == encoder_attn) self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) @@ -712,10 +711,7 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa_explicit = EncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue( - model_sdpa_explicit.config._attn_implementation - == {"encoder": encoder_attn, "decoder": decoder_attn} - ) + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") else: with self.assertRaises(ValueError): model_sdpa_explicit = EncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") @@ -726,7 +722,7 @@ def test_sdpa_can_dispatch_composite_models(self): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) + self.assertTrue(model_eager.config._attn_implementation == "eager") self.assertTrue(model_eager.encoder.config._attn_implementation == "eager") self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index 075f74f5c724..c94119db34fd 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -342,19 +342,13 @@ def test_sdpa_can_dispatch_composite_models(self): vision_attn = None if model.vision_model._supports_sdpa else "eager" perceiver_attn = None if model.connector.perceiver_resampler._supports_sdpa else "eager" - self.assertTrue( - model_sdpa.config._attn_implementation - == {"text_config": None, "perceiver_config": None, "vision_config": None} - ) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.vision_model.config._attn_implementation == vision_attn) self.assertTrue(model_sdpa.connector.perceiver_resampler.config._attn_implementation == perceiver_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) - self.assertTrue( - model_eager.config._attn_implementation - == {"text_config": "eager", "perceiver_config": "eager", "vision_config": "eager"} - ) + self.assertTrue(model_eager.config._attn_implementation == "eager") self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") self.assertTrue(model_sdpa.connector.perceiver_resampler.config._attn_implementation == "eager") diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 056d2d8c75c7..a769c85d2045 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -558,29 +558,23 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) - text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" - vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" - qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" + vision_attn = "sdpa" if model.vision_model._supports_sdpa else "eager" + qformer_attn = "sdpa" if model.qformer._supports_sdpa else "eager" # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue( - model_sdpa.config._attn_implementation - == {"text_config": None, "vision_config": None, "qformer_config": None} - ) - self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) - self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) - self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model.language_model.config._attn_implementation == text_attn) + self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) - self.assertTrue( - model_eager.config._attn_implementation - == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} - ) - self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.qformer.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 08f13fdaa6bb..caa29765059f 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -579,29 +579,23 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) - text_attn = "sdpa" if getattr(model, "language_model")._supports_sdpa else "eager" - vision_attn = "sdpa" if getattr(model, "vision_model")._supports_sdpa else "eager" - qformer_attn = "sdpa" if getattr(model, "qformer")._supports_sdpa else "eager" + text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" + vision_attn = "sdpa" if model.vision_model._supports_sdpa else "eager" + qformer_attn = "sdpa" if model.qformer._supports_sdpa else "eager" # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue( - model_sdpa.config._attn_implementation - == {"text_config": None, "vision_config": None, "qformer_config": None} - ) - self.assertTrue(getattr(model, "language_model").config._attn_implementation == text_attn) - self.assertTrue(getattr(model, "vision_model").config._attn_implementation == vision_attn) - self.assertTrue(getattr(model, "qformer").config._attn_implementation == qformer_attn) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model.language_model.config._attn_implementation == text_attn) + self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) + self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) - self.assertTrue( - model_eager.config._attn_implementation - == {"text_config": "eager", "vision_config": "eager", "qformer_config": "eager"} - ) - self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "vision_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "qformer").config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.qformer.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 004d04f3efe0..8f36d8c35477 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -1737,24 +1737,14 @@ def test_sdpa_can_dispatch_composite_models(self): self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) - self.assertTrue( - model_sdpa.config._attn_implementation - == { - "audio_encoder": None, - "text_encoder": None, - "decoder": None, - } - ) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") - self.assertTrue( - model_eager.config._attn_implementation - == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} - ) + self.assertTrue(model_eager.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): if "SdpaAttention" in submodule.__class__.__name__: diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index fd6abd249197..0cf0b7907a35 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -1723,24 +1723,14 @@ def test_sdpa_can_dispatch_composite_models(self): self.assertTrue(model_sdpa.audio_encoder.config._attn_implementation == audio_encoder_attn) self.assertTrue(model_sdpa.text_encoder.config._attn_implementation == text_encoder_attn) self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) - self.assertTrue( - model_sdpa.config._attn_implementation - == { - "audio_encoder": None, - "text_encoder": None, - "decoder": None, - } - ) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) self.assertTrue(model_eager.audio_encoder.config._attn_implementation == "eager") self.assertTrue(model_eager.text_encoder.config._attn_implementation == "eager") self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") - self.assertTrue( - model_eager.config._attn_implementation - == {"audio_encoder": "eager", "text_encoder": "eager", "decoder": "eager"} - ) + self.assertTrue(model_eager.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): if "SdpaAttention" in submodule.__class__.__name__: diff --git a/tests/models/qwen2_audio/test_modeling_qwen2_audio.py b/tests/models/qwen2_audio/test_modeling_qwen2_audio.py index 819c3ed46961..314f870f5d90 100644 --- a/tests/models/qwen2_audio/test_modeling_qwen2_audio.py +++ b/tests/models/qwen2_audio/test_modeling_qwen2_audio.py @@ -186,24 +186,20 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) - audio_model_sdpa = getattr(model, "audio_tower") - language_model_sdpa = getattr(model, "language_model") - text_attn = "sdpa" if language_model_sdpa._supports_sdpa else "eager" - vision_attn = "sdpa" if audio_model_sdpa._supports_sdpa else "eager" + text_attn = "sdpa" if model.language_model._supports_sdpa else "eager" + vision_attn = "sdpa" if model.audio_tower._supports_sdpa else "eager" # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "audio_config": None}) - self.assertTrue(language_model_sdpa.config._attn_implementation == text_attn) - self.assertTrue(audio_model_sdpa.config._attn_implementation == vision_attn) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model.language_model.config._attn_implementation == text_attn) + self.assertTrue(model.audio_tower.config._attn_implementation == vision_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "audio_config": "eager"} - ) - self.assertTrue(getattr(model_eager, "language_model").config._attn_implementation == "eager") - self.assertTrue(getattr(model_eager, "audio_tower").config._attn_implementation == "eager") + self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_eager.language_model.config._attn_implementation == "eager") + self.assertTrue(model_eager.audio_tower.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/siglip/test_modeling_siglip.py b/tests/models/siglip/test_modeling_siglip.py index 40704b740d76..2fe06b1511a4 100644 --- a/tests/models/siglip/test_modeling_siglip.py +++ b/tests/models/siglip/test_modeling_siglip.py @@ -99,14 +99,8 @@ def test_sdpa_can_dispatch_composite_models(self): self.assertTrue(model_eager.vision_model.config._attn_implementation == "eager") self.assertTrue(model_eager.text_model.config._attn_implementation == "eager") - if hasattr(model_sdpa.config, "text_config"): - self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} - ) - else: - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - self.assertTrue(model_eager.config._attn_implementation == "eager") + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") + self.assertTrue(model_eager.config._attn_implementation == "eager") for name, submodule in model_eager.named_modules(): class_name = submodule.__class__.__name__ diff --git a/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py b/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py index bf4697afaed2..6e0b7fa9782f 100644 --- a/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py +++ b/tests/models/speech_encoder_decoder/test_modeling_speech_encoder_decoder.py @@ -462,11 +462,10 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 - # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - self.assertTrue(model_sdpa.config._attn_implementation == {"encoder": None, "decoder": None}) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.encoder.config._attn_implementation == encoder_attn) self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) @@ -477,10 +476,7 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa_explicit = SpeechEncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue( - model_sdpa_explicit.config._attn_implementation - == {"encoder": encoder_attn, "decoder": decoder_attn} - ) + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") else: with self.assertRaises(ValueError): model_sdpa_explicit = SpeechEncoderDecoderModel.from_pretrained( @@ -493,7 +489,7 @@ def test_sdpa_can_dispatch_composite_models(self): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) + self.assertTrue(model_eager.config._attn_implementation == "eager") self.assertTrue(model_eager.encoder.config._attn_implementation == "eager") self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") diff --git a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py index bbad7aa5cea4..7def8a9ac965 100644 --- a/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py +++ b/tests/models/vision_encoder_decoder/test_modeling_vision_encoder_decoder.py @@ -403,11 +403,10 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa = model_sdpa.eval().to(torch_device) # see https://github.com/huggingface/transformers/pull/32238 - # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) encoder_attn = "sdpa" if model.encoder._supports_sdpa else "eager" decoder_attn = "sdpa" if model.decoder._supports_sdpa else "eager" - self.assertTrue(model_sdpa.config._attn_implementation == {"encoder": None, "decoder": None}) + self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model_sdpa.encoder.config._attn_implementation == encoder_attn) self.assertTrue(model_sdpa.decoder.config._attn_implementation == decoder_attn) @@ -418,10 +417,7 @@ def test_sdpa_can_dispatch_composite_models(self): model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained(tmpdirname, attn_implementation="sdpa") model_sdpa_explicit = model_sdpa_explicit.eval().to(torch_device) - self.assertTrue( - model_sdpa_explicit.config._attn_implementation - == {"encoder": encoder_attn, "decoder": decoder_attn} - ) + self.assertTrue(model_sdpa_explicit.config._attn_implementation == "sdpa") else: with self.assertRaises(ValueError): model_sdpa_explicit = VisionEncoderDecoderModel.from_pretrained( @@ -434,7 +430,7 @@ def test_sdpa_can_dispatch_composite_models(self): ) model_eager = model_eager.eval().to(torch_device) - self.assertTrue(model_eager.config._attn_implementation == {"encoder": "eager", "decoder": "eager"}) + self.assertTrue(model_eager.config._attn_implementation == "eager") self.assertTrue(model_eager.encoder.config._attn_implementation == "eager") self.assertTrue(model_eager.decoder.config._attn_implementation == "eager") diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 09c7d43b3491..b8ecf2152685 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3801,7 +3801,11 @@ def test_attn_implementation_composite_models(self): config._attn_implementation = attn_implementation_per_subconfig model = model_class(config) - self.assertTrue(model.config._attn_implementation == attn_implementation_per_subconfig) + for key in model.config: + if isinstance(getattr(model.config, key), PretrainedConfig): + sub_config = getattr(model.config, key) + self.assertTrue(sub_config._attn_implementation == "eager") + for name, submodule in model.named_modules(): class_name = submodule.__class__.__name__ if ( @@ -3832,6 +3836,7 @@ def test_sdpa_can_dispatch_non_composite_models(self): model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) + print(model_sdpa.config._attn_implementation) self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") @@ -3891,15 +3896,11 @@ def test_sdpa_can_dispatch_composite_models(self): # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.config._attn_implementation == {"text_config": None, "vision_config": None}) self.assertTrue(language_model_sdpa.config._attn_implementation == text_attn) self.assertTrue(vision_model_sdpa.config._attn_implementation == vision_attn) model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") model_eager = model_eager.eval().to(torch_device) - self.assertTrue( - model_eager.config._attn_implementation == {"text_config": "eager", "vision_config": "eager"} - ) self.assertTrue(getattr(model_eager, language_model_name).config._attn_implementation == "eager") self.assertTrue(getattr(model_eager, vision_model_name).config._attn_implementation == "eager") @@ -4505,7 +4506,10 @@ def test_flash_attn_2_can_dispatch_composite_models(self): model_fa2 = model_class.from_pretrained( tmpdirname, torch_dtype=torch_dtype, attn_implementation="flash_attention_2" ) - self.assertTrue("flash_attention_2" in model_fa2.config._attn_implementation.values()) + for key in model_fa2.config: + if isinstance(getattr(model_fa2.config, key), PretrainedConfig): + sub_config = getattr(model_fa2.config, key) + self.assertTrue(sub_config._attn_implementation == "flash_attention_2") has_fa2 = False for name, submodule in model_fa2.named_modules(): From 6dcf21a008d38971aa221f406b5d10443af35f35 Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 8 Oct 2024 19:37:02 +0200 Subject: [PATCH 62/68] fix tests --- src/transformers/modeling_utils.py | 11 ++++++----- .../instructblipvideo/modeling_instructblipvideo.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 3a42b6c8f645..2f251c67ebd4 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1419,7 +1419,7 @@ def __init__(self, config: PretrainedConfig, *inputs, **kwargs): f"`model = {self.__class__.__name__}.from_pretrained(PRETRAINED_MODEL_NAME)`" ) # Save config and origin of the pretrained weights if given in model - if not config._attn_implementation_autoset: + if not getattr(config, "_attn_implementation_autoset", False): config = self._autoset_attn_implementation( config, torch_dtype=torch.get_default_dtype(), check_device_map=False ) @@ -1518,7 +1518,7 @@ def _from_config(cls, config, **kwargs): attn_implementation = None config._attn_implementation = kwargs.pop("attn_implementation", attn_implementation) - if not config._attn_implementation_autoset: + if not getattr(config, "_attn_implementation_autoset", False): config = cls._autoset_attn_implementation( config, use_flash_attention_2=use_flash_attention_2, @@ -3933,9 +3933,10 @@ def from_pretrained( init_contexts.append(init_empty_weights()) config = copy.deepcopy(config) # We do not want to modify the config inplace in from_pretrained. - config = cls._autoset_attn_implementation( - config, use_flash_attention_2=use_flash_attention_2, torch_dtype=torch_dtype, device_map=device_map - ) + if not getattr(config, "_attn_implementation_autoset", False): + config = cls._autoset_attn_implementation( + config, use_flash_attention_2=use_flash_attention_2, torch_dtype=torch_dtype, device_map=device_map + ) with ContextManagers(init_contexts): # Let's make sure we don't run the init function of buffer modules diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 250a0a41b442..dd4edb795769 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -1296,8 +1296,8 @@ def forward( class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel, GenerationMixin): config_class = InstructBlipVideoConfig main_input_name = "pixel_values" - _supports_flash_attn_2 = False - _supports_sdpa = False + _supports_flash_attn_2 = True + _supports_sdpa = True def __init__(self, config: InstructBlipVideoConfig): super().__init__(config) From b93b79a7e0aa3ca1cc28c7bd6ff5ea4d0808e6cd Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 9 Oct 2024 13:33:41 +0200 Subject: [PATCH 63/68] fix tests --- src/transformers/modeling_utils.py | 9 +++++++- .../modeling_encoder_decoder.py | 22 +++++++++---------- .../models/idefics2/test_modeling_idefics2.py | 5 ----- tests/models/kosmos2/test_modeling_kosmos2.py | 8 ------- .../llava_next/test_modeling_llava_next.py | 3 ++- .../test_modeling_llava_next_video.py | 3 ++- .../models/musicgen/test_modeling_musicgen.py | 4 ++-- .../test_modeling_musicgen_melody.py | 2 +- .../video_llava/test_modeling_video_llava.py | 3 ++- tests/test_modeling_common.py | 3 +-- utils/modular_model_converter.py | 8 +++---- 11 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 22223a9764b2..1e4f780f688d 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1500,6 +1500,10 @@ def _from_config(cls, config, **kwargs): torch_dtype (`torch.dtype`, *optional*): Override the default `torch.dtype` and load the model under this dtype. """ + + # when we init a model from within another model (e.g. VLMs) and dispatch on FA2 + # a warning is raised that dtype should be fp16. Since we never pass dtype from within + # modeling code, we can try to infer it here same way as done in `from_pretrained` torch_dtype = kwargs.pop("torch_dtype", torch.get_default_dtype()) use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False) @@ -1591,7 +1595,7 @@ def _autoset_attn_implementation( # for all sub-models. # Below we check if a config is composite and manually prepare a dict of attn impl if not already passed as a dict. # Later each sub-module will dispatch with its own attn impl, by calling `XXXModel._from_config(config.text_config)` - # If any of sub-modules doesm't support requested attn, an error will be raised. See https://github.com/huggingface/transformers/pull/32238 + # If any of sub-modules doesn't support requested attn, an error will be raised. See https://github.com/huggingface/transformers/pull/32238 for key in config: if isinstance(getattr(config, key), PretrainedConfig): sub_config = getattr(config, key) @@ -2785,6 +2789,9 @@ def save_pretrained( # Attach architecture to the config model_to_save.config.architectures = [model_to_save.__class__.__name__] + # Unset attn implementation so it can be set to another one when loading back + model_to_save.config._attn_implementation_autoset = False + # If we have a custom model, we copy the file defining it in the folder and set the attributes so it can be # loaded from the Hub. if self._auto_class is not None: diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index 304f02750b64..051c5b448ead 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -221,20 +221,20 @@ def __init__( self.encoder = encoder self.decoder = decoder - # if self.encoder.config.to_dict() != self.config.encoder.to_dict(): - # logger.warning( - # f"Config of the encoder: {self.encoder.__class__} is overwritten by shared encoder config:" - # f" {self.config.encoder}" - # ) - # if self.decoder.config.to_dict() != self.config.decoder.to_dict(): - # logger.warning( - # f"Config of the decoder: {self.decoder.__class__} is overwritten by shared decoder config:" - # f" {self.config.decoder}" - # ) + if self.encoder.config.to_dict() != self.config.encoder.to_dict(): + logger.warning( + f"Config of the encoder: {self.encoder.__class__} is overwritten by shared encoder config:" + f" {self.config.encoder}" + ) + if self.decoder.config.to_dict() != self.config.decoder.to_dict(): + logger.warning( + f"Config of the decoder: {self.decoder.__class__} is overwritten by shared decoder config:" + f" {self.config.decoder}" + ) # make sure that the individual model's config refers to the shared config # so that the updates to the config will be synced - # update `_attn_implementation` because the attn is set a a deepcopied config within PreTrainedMolde + # update `_attn_implementation` because the attn is set in a deepcopied config within PreTrainedModel self.config.encoder._attn_implementation = self.encoder.config._attn_implementation self.config.decoder._attn_implementation = self.decoder.config._attn_implementation self.encoder.config = self.config.encoder diff --git a/tests/models/idefics2/test_modeling_idefics2.py b/tests/models/idefics2/test_modeling_idefics2.py index c94119db34fd..0ece281353d6 100644 --- a/tests/models/idefics2/test_modeling_idefics2.py +++ b/tests/models/idefics2/test_modeling_idefics2.py @@ -380,11 +380,6 @@ class Idefics2ForConditionalGenerationModelTest(GenerationTesterMixin, ModelTest test_head_masking = False test_torchscript = False - # We define this flag here because in VLMs these flags depend on which LM/vision models are used - # So we can't know if SDPA is supported before starting to load the model - # This flag is used by tests and is set to False because LM/vision models used in tests don't support SDPA - supports_sdpa = False - def setUp(self): self.model_tester = Idefics2VisionText2TextModelTester(self) self.config_tester = ConfigTester(self, config_class=Idefics2Config, has_text_modality=False) diff --git a/tests/models/kosmos2/test_modeling_kosmos2.py b/tests/models/kosmos2/test_modeling_kosmos2.py index 0dd7b478b9da..6fdc5b0e33f4 100644 --- a/tests/models/kosmos2/test_modeling_kosmos2.py +++ b/tests/models/kosmos2/test_modeling_kosmos2.py @@ -517,14 +517,6 @@ def _create_and_check_torchscript(self, config, inputs_dict): # (Even with this call, there are still memory leak by ~0.04MB) self.clear_torch_jit_class_registry() - @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") - def test_sdpa_can_dispatch_composite_models(self): - pass - - @unittest.skip("Kosmos2 doesn't support attn implementation flag at all and has only eager layers") - def test_flash_attn_2_can_dispatch_composite_models(self): - pass - # We will verify our results on an image of cute cats def prepare_img(): diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index eaa6e6ae53ba..2bfcc9f64f35 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -34,6 +34,7 @@ torch_device, ) +from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -208,7 +209,7 @@ def create_and_check_llava_next_model_fp16_autocast_forward( @require_torch -class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): +class LlavaNextForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): """ Model tester for `LlavaNextForConditionalGeneration`. """ diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 242e35b416e2..726d4148fed6 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -35,6 +35,7 @@ torch_device, ) +from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -226,7 +227,7 @@ def create_and_check_llava_next_video_model_fp16_autocast_forward( @require_torch -class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): +class LlavaNextVideoForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): """ Model tester for `LlavaNextVideoForConditionalGeneration`. """ diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 8f36d8c35477..3857319082e1 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -596,7 +596,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: + if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): @@ -1766,7 +1766,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: + if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index 0cf0b7907a35..5e13e3d24536 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -600,7 +600,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: + if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index 963716ef32c8..32efcecbc5fe 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -30,6 +30,7 @@ ) from transformers.testing_utils import require_bitsandbytes, require_torch, require_torch_gpu, slow, torch_device +from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor @@ -194,7 +195,7 @@ def prepare_config_and_inputs_for_batched_test(self): @require_torch -class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, unittest.TestCase): +class VideoLlavaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): """ Model tester for `VideoLlavaForConditionalGeneration`. """ diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index e23cd243e27e..933881e472b5 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -3990,7 +3990,6 @@ def test_sdpa_can_dispatch_non_composite_models(self): model_sdpa = model_class.from_pretrained(tmpdirname) model_sdpa = model_sdpa.eval().to(torch_device) - print(model_sdpa.config._attn_implementation) self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") model_eager = model_class.from_pretrained(tmpdirname, attn_implementation="eager") @@ -4079,7 +4078,7 @@ def test_eager_matches_sdpa_inference(self, torch_dtype: str): if not self.has_attentions: self.skipTest(reason="Model architecture does not support attentions") - if not self.all_model_classes[0]._supports_sdpa and not self._is_composite: + if not self.all_model_classes[0]._supports_sdpa: self.skipTest(f"{self.all_model_classes[0].__name__} does not support SDPA") if torch_dtype == "float16" and not is_torch_fp16_available_on_device(torch_device): diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index 599dc70e17e8..fe4f52d1f61b 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -18,7 +18,7 @@ import os import re from collections import defaultdict, deque -from typing import Dict, List, Set +from typing import Dict, List, Set, Optional import libcst as cst from check_copies import run_ruff @@ -623,7 +623,7 @@ def get_new_part(class_name, base_class): return snake_case -def find_all_dependencies(function: str, dependency_mapping: dict[str, set]): +def find_all_dependencies(function: str, dependency_mapping: Dict[str, set]): """Return all the dependencies of the given top-level function. Given the following structure in the `modular_xxx.py` file: ``` def foo1(): @@ -1001,8 +1001,8 @@ def _maybe_add_function_to_body( top_level_function: str, body: dict, function_node: cst.FunctionDef, - matching_callers: set | None = None, - parent: str | None = None, + matching_callers: Optional[set] = None, + parent: Optional[str] = None, ) -> bool: """Check if the `top_level_function` should be added to the body (i.e. it is not already present, and `matching_callers` is not empy, or `parent`is provided). If it should be added, do it (in the correct location, just before its caller) and return From 702dacf8d388dadd874c6b446cdb4addeea22a01 Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 10 Oct 2024 14:11:26 +0200 Subject: [PATCH 64/68] update --- src/transformers/modeling_utils.py | 1 + tests/utils/test_configuration_utils.py | 1 + utils/modular_model_converter.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 1e4f780f688d..84e038d19c46 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2791,6 +2791,7 @@ def save_pretrained( # Unset attn implementation so it can be set to another one when loading back model_to_save.config._attn_implementation_autoset = False + model_to_save.config._attn_implementation_internal = None # If we have a custom model, we copy the file defining it in the folder and set the attributes so it can be # loaded from the Hub. diff --git a/tests/utils/test_configuration_utils.py b/tests/utils/test_configuration_utils.py index d2701bf35e66..35a651d0e598 100644 --- a/tests/utils/test_configuration_utils.py +++ b/tests/utils/test_configuration_utils.py @@ -228,6 +228,7 @@ def test_config_common_kwargs_is_complete(self): "_name_or_path", "_commit_hash", "_attn_implementation_internal", + "_attn_implementation_autoset", "transformers_version", ], ) diff --git a/utils/modular_model_converter.py b/utils/modular_model_converter.py index fe4f52d1f61b..0ae641d4e502 100644 --- a/utils/modular_model_converter.py +++ b/utils/modular_model_converter.py @@ -18,7 +18,7 @@ import os import re from collections import defaultdict, deque -from typing import Dict, List, Set, Optional +from typing import Dict, List, Optional, Set import libcst as cst from check_copies import run_ruff From 0616732c2cdbc1b7f472809d513a3d4e6f7343ac Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 10 Oct 2024 14:49:33 +0200 Subject: [PATCH 65/68] another update --- src/transformers/modeling_utils.py | 1 - src/transformers/models/blip_2/modeling_blip_2.py | 12 ++---------- .../models/instructblip/modeling_instructblip.py | 6 +----- .../instructblipvideo/modeling_instructblipvideo.py | 6 +----- tests/models/blip_2/test_modeling_blip_2.py | 2 -- .../instructblip/test_modeling_instructblip.py | 1 - .../test_modeling_instructblipvideo.py | 1 - 7 files changed, 4 insertions(+), 25 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 84e038d19c46..1e4f780f688d 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2791,7 +2791,6 @@ def save_pretrained( # Unset attn implementation so it can be set to another one when loading back model_to_save.config._attn_implementation_autoset = False - model_to_save.config._attn_implementation_internal = None # If we have a custom model, we copy the file defining it in the folder and set the attributes so it can be # loaded from the Hub. diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index bb3370033972..eba82cd1b3c8 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -410,8 +410,6 @@ class Blip2PreTrainedModel(PreTrainedModel): config_class = Blip2Config base_model_prefix = "blip" supports_gradient_checkpointing = True - _supports_flash_attn_2 = False - _supports_sdpa = False _no_split_modules = [ "Blip2Attention", @@ -1448,13 +1446,10 @@ class Blip2Model(Blip2PreTrainedModel): config_class = Blip2Config main_input_name = "pixel_values" - _supports_flash_attn_2 = True - _supports_sdpa = True - def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel._from_config(config.vision_config) + self.vision_model = Blip2VisionModel(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = Blip2QFormerModel(config.qformer_config) @@ -2012,13 +2007,10 @@ class Blip2ForConditionalGeneration(Blip2PreTrainedModel, GenerationMixin): config_class = Blip2Config main_input_name = "pixel_values" - _supports_flash_attn_2 = True - _supports_sdpa = True - def __init__(self, config: Blip2Config): super().__init__(config) - self.vision_model = Blip2VisionModel._from_config(config.vision_config) + self.vision_model = Blip2VisionModel(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = Blip2QFormerModel(config.qformer_config) diff --git a/src/transformers/models/instructblip/modeling_instructblip.py b/src/transformers/models/instructblip/modeling_instructblip.py index 73b175a7c10f..5cce774ce071 100644 --- a/src/transformers/models/instructblip/modeling_instructblip.py +++ b/src/transformers/models/instructblip/modeling_instructblip.py @@ -315,8 +315,6 @@ class InstructBlipPreTrainedModel(PreTrainedModel): config_class = InstructBlipConfig base_model_prefix = "blip" supports_gradient_checkpointing = True - _supports_flash_attn_2 = False - _supports_sdpa = False _no_split_modules = [ "InstructBlipQFormerEmbeddings", @@ -1289,13 +1287,11 @@ def forward( class InstructBlipForConditionalGeneration(InstructBlipPreTrainedModel, GenerationMixin): config_class = InstructBlipConfig main_input_name = "pixel_values" - _supports_flash_attn_2 = True - _supports_sdpa = True def __init__(self, config: InstructBlipConfig): super().__init__(config) - self.vision_model = InstructBlipVisionModel._from_config(config.vision_config) + self.vision_model = InstructBlipVisionModel(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = InstructBlipQFormerModel(config.qformer_config) diff --git a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py index 14cdc147c363..c9f12391666c 100644 --- a/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py +++ b/src/transformers/models/instructblipvideo/modeling_instructblipvideo.py @@ -317,8 +317,6 @@ class InstructBlipVideoPreTrainedModel(PreTrainedModel): config_class = InstructBlipVideoConfig base_model_prefix = "blip" supports_gradient_checkpointing = True - _supports_flash_attn_2 = False - _supports_sdpa = False _no_split_modules = [ "InstructBlipVideoQFormerEmbeddings", @@ -1283,13 +1281,11 @@ def forward( class InstructBlipVideoForConditionalGeneration(InstructBlipVideoPreTrainedModel, GenerationMixin): config_class = InstructBlipVideoConfig main_input_name = "pixel_values" - _supports_flash_attn_2 = True - _supports_sdpa = True def __init__(self, config: InstructBlipVideoConfig): super().__init__(config) - self.vision_model = InstructBlipVideoVisionModel._from_config(config.vision_config) + self.vision_model = InstructBlipVideoVisionModel(config.vision_config) self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) self.qformer = InstructBlipVideoQFormerModel(config.qformer_config) diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 27b616767cbe..933d97a34bcb 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -523,7 +523,6 @@ def test_sdpa_can_dispatch_composite_models(self): # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model.language_model.config._attn_implementation == text_attn) self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) @@ -858,7 +857,6 @@ def test_sdpa_can_dispatch_composite_models(self): # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model.language_model.config._attn_implementation == text_attn) self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index a769c85d2045..5182ac20cd99 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -564,7 +564,6 @@ def test_sdpa_can_dispatch_composite_models(self): # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model.language_model.config._attn_implementation == text_attn) self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index caa29765059f..298c7a8d7ff4 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -585,7 +585,6 @@ def test_sdpa_can_dispatch_composite_models(self): # `None` as it is the requested one which will be assigned to each sub-config # Sub-model will dispatch to SDPA if it can (checked below that `SDPA` layers are present) - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") self.assertTrue(model.language_model.config._attn_implementation == text_attn) self.assertTrue(model.vision_model.config._attn_implementation == vision_attn) self.assertTrue(model.qformer.config._attn_implementation == qformer_attn) From ebf15efc564c641dc926b266dd82a738c54cf7af Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 11 Oct 2024 10:38:43 +0200 Subject: [PATCH 66/68] fix tests --- docs/source/en/perf_infer_gpu_one.md | 16 ++++++++++++++++ tests/models/blip_2/test_modeling_blip_2.py | 11 +++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/source/en/perf_infer_gpu_one.md b/docs/source/en/perf_infer_gpu_one.md index cf2dac617ffa..62e75c350fa0 100644 --- a/docs/source/en/perf_infer_gpu_one.md +++ b/docs/source/en/perf_infer_gpu_one.md @@ -77,6 +77,7 @@ FlashAttention-2 is currently supported for the following architectures: * [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel) * [OLMoE](https://huggingface.co/docs/transformers/model_doc/olmoe#transformers.OlmoeModel) * [OPT](https://huggingface.co/docs/transformers/model_doc/opt#transformers.OPTModel) +* [PaliGemma](https://huggingface.co/docs/transformers/model_doc/paligemma#transformers.PaliGemmaForConditionalGeneration) * [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel) * [Phi3](https://huggingface.co/docs/transformers/model_doc/phi3#transformers.Phi3Model) * [PhiMoE](https://huggingface.co/docs/transformers/model_doc/phimoe#transformers.PhimoeModel) @@ -86,6 +87,10 @@ FlashAttention-2 is currently supported for the following architectures: * [Qwen2Audio](https://huggingface.co/docs/transformers/model_doc/qwen2_audio#transformers.Qwen2AudioEncoder) * [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel) * [Qwen2VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl#transformers.Qwen2VLModel) +* [RAG](https://huggingface.co/docs/transformers/model_doc/rag#transformers.RagModel) +* [SpeechEncoderDecoder](https://huggingface.co/docs/transformers/model_doc/speech_encoder_decoder#transformers.SpeechEncoderDecoderModel) +* [VisionEncoderDecoder](https://huggingface.co/docs/transformers/model_doc/vision_encoder_decoder#transformers.VisionEncoderDecoderModel) +* [VisionTextDualEncoder](https://huggingface.co/docs/transformers/model_doc/vision_text_dual_encoder#transformers.VisionTextDualEncoderModel) * [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel) * [Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model) * [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel) @@ -222,6 +227,7 @@ For now, Transformers supports SDPA inference and training for the following arc * [Dinov2](https://huggingface.co/docs/transformers/en/model_doc/dinov2) * [DistilBert](https://huggingface.co/docs/transformers/model_doc/distilbert#transformers.DistilBertModel) * [Dpr](https://huggingface.co/docs/transformers/model_doc/dpr#transformers.DprReader) +* [EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder_decoder#transformers.EncoderDecoderModel) * [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel) * [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel) * [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model) @@ -230,11 +236,16 @@ For now, Transformers supports SDPA inference and training for the following arc * [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel) * [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel) * [Idefics](https://huggingface.co/docs/transformers/model_doc/idefics#transformers.IdeficsModel) +* [Idefics2](https://huggingface.co/docs/transformers/model_doc/idefics2#transformers.Idefics2Model) +* [Idefics3](https://huggingface.co/docs/transformers/model_doc/idefics3#transformers.Idefics3Model) * [Granite](https://huggingface.co/docs/transformers/model_doc/granite#transformers.GraniteModel) * [GraniteMoe](https://huggingface.co/docs/transformers/model_doc/granitemoe#transformers.GraniteMoeModel) * [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel) * [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel) * [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel) +* [Llava](https://huggingface.co/docs/transformers/model_doc/llava) +* [Llava-NeXT](https://huggingface.co/docs/transformers/model_doc/llava_next) +* [Llava-NeXT-Video](https://huggingface.co/docs/transformers/model_doc/llava_next_video) * [LLaVA-Onevision](https://huggingface.co/docs/transformers/model_doc/llava_onevision) * [M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100#transformers.M2M100Model) * [Mimi](https://huggingface.co/docs/transformers/model_doc/mimi) @@ -273,10 +284,15 @@ For now, Transformers supports SDPA inference and training for the following arc * [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel) * [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel) * [Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron) +* [SpeechEncoderDecoder](https://huggingface.co/docs/transformers/model_doc/speech_encoder_decoder#transformers.SpeechEncoderDecoderModel) +* [VideoLlava](https://huggingface.co/docs/transformers/model_doc/video_llava) +* [VipLlava](https://huggingface.co/docs/transformers/model_doc/vipllava) +* [VisionEncoderDecoder](https://huggingface.co/docs/transformers/model_doc/vision_encoder_decoder#transformers.VisionEncoderDecoderModel) * [ViT](https://huggingface.co/docs/transformers/model_doc/vit#transformers.ViTModel) * [ViTHybrid](https://huggingface.co/docs/transformers/model_doc/vit_hybrid#transformers.ViTHybridModel) * [ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae#transformers.ViTMAEModel) * [ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn#transformers.ViTMSNModel) +* [VisionTextDualEncoder](https://huggingface.co/docs/transformers/model_doc/vision_text_dual_encoder#transformers.VisionTextDualEncoderModel) * [VideoMAE](https://huggingface.co/docs/transformers/model_doc/videomae#transformers.VideoMAEModell) * [wav2vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model) * [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel) diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 933d97a34bcb..e5d04bd85a34 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -781,9 +781,16 @@ class Blip2ModelTest(ModelTesterMixin, PipelineTesterMixin, GenerationTesterMixi # TODO: Fix the failed tests def is_pipeline_test_to_skip( - self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + self, + pipeline_test_case_name, + config_class, + model_architecture, + tokenizer_name, + image_processor_name, + feature_extractor_name, + processor_name, ): - if pipeline_test_casse_name == "VisualQuestionAnsweringPipelineTests": + if pipeline_test_case_name == "VisualQuestionAnsweringPipelineTests": # Get `RuntimeError: "LayerNormKernelImpl" not implemented for 'Half'`. return True From 24a9fc534adbf42c0875474fff122b9ccc7c079d Mon Sep 17 00:00:00 2001 From: raushan Date: Mon, 21 Oct 2024 13:42:35 +0200 Subject: [PATCH 67/68] fix copies --- src/transformers/models/glm/modeling_glm.py | 41 +++------------------ 1 file changed, 5 insertions(+), 36 deletions(-) diff --git a/src/transformers/models/glm/modeling_glm.py b/src/transformers/models/glm/modeling_glm.py index 9815dbc78992..a458c02a6fed 100644 --- a/src/transformers/models/glm/modeling_glm.py +++ b/src/transformers/models/glm/modeling_glm.py @@ -25,7 +25,6 @@ import torch import torch.nn as nn import torch.utils.checkpoint -from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache, StaticCache @@ -921,6 +920,7 @@ def _prepare_4d_causal_attention_mask_with_cache_position( device: torch.device, cache_position: torch.Tensor, batch_size: int, + **kwargs, ): """ Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape @@ -1071,18 +1071,7 @@ def forward( loss = None if labels is not None: - # Upcast to float if we need to compute the loss to avoid potential precision issues - logits = logits.float() - # Shift so that tokens < n predict n - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - # Flatten the tokens - loss_fct = CrossEntropyLoss() - shift_logits = shift_logits.view(-1, self.config.vocab_size) - shift_labels = shift_labels.view(-1) - # Enable model parallelism - shift_labels = shift_labels.to(shift_logits.device) - loss = loss_fct(shift_logits, shift_labels) + loss = self.loss_function(logits, labels, self.vocab_size) if not return_dict: output = (logits,) + outputs[1:] @@ -1186,27 +1175,8 @@ def forward( loss = None if labels is not None: - labels = labels.to(logits.device) - if self.config.problem_type is None: - if self.num_labels == 1: - self.config.problem_type = "regression" - elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): - self.config.problem_type = "single_label_classification" - else: - self.config.problem_type = "multi_label_classification" - - if self.config.problem_type == "regression": - loss_fct = MSELoss() - if self.num_labels == 1: - loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) - else: - loss = loss_fct(pooled_logits, labels) - elif self.config.problem_type == "single_label_classification": - loss_fct = CrossEntropyLoss() - loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) - elif self.config.problem_type == "multi_label_classification": - loss_fct = BCEWithLogitsLoss() - loss = loss_fct(pooled_logits, labels) + loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config) + if not return_dict: output = (pooled_logits,) + transformer_outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1289,8 +1259,7 @@ def forward( loss = None if labels is not None: - loss_fct = CrossEntropyLoss() - loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + loss = self.loss_function(logits, labels, self.config) if not return_dict: output = (logits,) + outputs[2:] From 4eed237d933af73f7a5c8e1718ab415791747194 Mon Sep 17 00:00:00 2001 From: raushan Date: Mon, 21 Oct 2024 16:35:10 +0200 Subject: [PATCH 68/68] fix tests --- .../models/paligemma/modeling_paligemma.py | 5 + tests/models/llava/test_modeling_llava.py | 10 + .../llava_next/test_modeling_llava_next.py | 10 + .../test_modeling_llava_next_video.py | 10 + .../test_modeling_llava_onevision.py | 10 + .../models/musicgen/test_modeling_musicgen.py | 105 +++++- .../test_modeling_musicgen_melody.py | 319 ++++++------------ .../paligemma/test_modeling_paligemma.py | 10 + .../video_llava/test_modeling_video_llava.py | 10 + .../models/vipllava/test_modeling_vipllava.py | 10 + tests/test_modeling_common.py | 4 +- 11 files changed, 273 insertions(+), 230 deletions(-) diff --git a/src/transformers/models/paligemma/modeling_paligemma.py b/src/transformers/models/paligemma/modeling_paligemma.py index a24defeec8e9..ffb4b7435f2a 100644 --- a/src/transformers/models/paligemma/modeling_paligemma.py +++ b/src/transformers/models/paligemma/modeling_paligemma.py @@ -343,6 +343,11 @@ def tie_weights(self): def _update_causal_mask( self, attention_mask, token_type_ids, inputs_embeds, past_key_values, cache_position, is_training: bool = False ): + if self.config.text_config._attn_implementation == "flash_attention_2": + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + using_static_cache = isinstance(past_key_values, StaticCache) dtype = inputs_embeds.dtype min_dtype = torch.finfo(dtype).min diff --git a/tests/models/llava/test_modeling_llava.py b/tests/models/llava/test_modeling_llava.py index 70cfb6010dc1..405fad1bd31c 100644 --- a/tests/models/llava/test_modeling_llava.py +++ b/tests/models/llava/test_modeling_llava.py @@ -261,6 +261,16 @@ def test_sdpa_can_compile_dynamic(self): def test_sdpa_can_dispatch_on_flash(self): pass + @unittest.skip("FlashAttention only support fp16 and bf16 data type") + def test_flash_attn_2_fp32_ln(self): + pass + + @unittest.skip( + "VLMs need lots of steps to prepare images/mask correctly to get pad-free inputs. Can be tested as part of LLM test" + ) + def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): + pass + @require_torch class LlavaForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/models/llava_next/test_modeling_llava_next.py b/tests/models/llava_next/test_modeling_llava_next.py index 2bfcc9f64f35..6589bf14d24c 100644 --- a/tests/models/llava_next/test_modeling_llava_next.py +++ b/tests/models/llava_next/test_modeling_llava_next.py @@ -317,6 +317,16 @@ def test_sdpa_can_compile_dynamic(self): def test_sdpa_can_dispatch_on_flash(self): pass + @unittest.skip("FlashAttention only support fp16 and bf16 data type") + def test_flash_attn_2_fp32_ln(self): + pass + + @unittest.skip( + "VLMs need lots of steps to prepare images/mask correctly to get pad-free inputs. Can be tested as part of LLM test" + ) + def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): + pass + @require_torch class LlavaNextForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/models/llava_next_video/test_modeling_llava_next_video.py b/tests/models/llava_next_video/test_modeling_llava_next_video.py index 726d4148fed6..05fc8a49e1e9 100644 --- a/tests/models/llava_next_video/test_modeling_llava_next_video.py +++ b/tests/models/llava_next_video/test_modeling_llava_next_video.py @@ -341,6 +341,16 @@ def test_sdpa_can_compile_dynamic(self): def test_sdpa_can_dispatch_on_flash(self): pass + @unittest.skip("FlashAttention only support fp16 and bf16 data type") + def test_flash_attn_2_fp32_ln(self): + pass + + @unittest.skip( + "VLMs need lots of steps to prepare images/mask correctly to get pad-free inputs. Can be tested as part of LLM test" + ) + def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): + pass + @require_torch class LlavaNextVideoForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/models/llava_onevision/test_modeling_llava_onevision.py b/tests/models/llava_onevision/test_modeling_llava_onevision.py index fb890e072ea1..0a33898b6307 100644 --- a/tests/models/llava_onevision/test_modeling_llava_onevision.py +++ b/tests/models/llava_onevision/test_modeling_llava_onevision.py @@ -307,6 +307,16 @@ def test_training_gradient_checkpointing_use_reentrant_false(self): def test_assisted_decoding_with_num_logits_to_keep(self): pass + @unittest.skip("FlashAttention only support fp16 and bf16 data type") + def test_flash_attn_2_fp32_ln(self): + pass + + @unittest.skip( + "VLMs need lots of steps to prepare images/mask correctly to get pad-free inputs. Can be tested as part of LLM test" + ) + def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): + pass + @require_torch class LlavaOnevisionForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/models/musicgen/test_modeling_musicgen.py b/tests/models/musicgen/test_modeling_musicgen.py index 3857319082e1..438178bfc6fa 100644 --- a/tests/models/musicgen/test_modeling_musicgen.py +++ b/tests/models/musicgen/test_modeling_musicgen.py @@ -1405,7 +1405,7 @@ def test_save_load_fast_init_from_base(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence def test_flash_attn_2_inference_equivalence(self): for model_class in self.all_model_classes: if not model_class._supports_flash_attn_2: @@ -1417,7 +1417,9 @@ def test_flash_attn_2_inference_equivalence(self): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model_fa = model_class.from_pretrained( - tmpdirname, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + tmpdirname, + torch_dtype=torch.bfloat16, + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, ) model_fa.to(torch_device) @@ -1490,7 +1492,88 @@ def test_flash_attn_2_inference_equivalence(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence_right_padding + def test_flash_attn_2_conversion(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + if not model_class._supports_flash_attn_2: + self.skipTest(f"{model_class.__name__} does not support Flash Attention 2") + + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model = model_class.from_pretrained( + tmpdirname, + torch_dtype=torch.float16, + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, + ).to(torch_device) + + for _, module in model.named_modules(): + if "FlashAttention" in module.__class__.__name__: + return + + self.assertTrue(False, "FlashAttention2 modules not found in model") + + @require_torch_sdpa + @require_torch_gpu + @slow + def test_sdpa_can_dispatch_on_flash(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + torch.compiler.reset() + compute_capability = torch.cuda.get_device_capability() + major, _ = compute_capability + + if not torch.version.cuda or major < 8: + self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") + + for model_class in self.all_model_classes: + if not model_class._supports_sdpa: + self.skipTest(f"{model_class.__name__} does not support SDPA") + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + inputs_dict = self._prepare_for_class(inputs_dict, model_class) + if config.model_type in ["llava", "llava_next", "vipllava", "video_llava"]: + self.skipTest( + reason="Llava-like models currently (transformers==4.39.1) requires an attention_mask input" + ) + if config.model_type in ["paligemma"]: + self.skipTest( + "PaliGemma-like models currently (transformers==4.41.0) requires an attention_mask input" + ) + if config.model_type in ["idefics", "idefics2", "idefics3"]: + self.skipTest(reason="Idefics currently (transformers==4.39.1) requires an image_attention_mask input") + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model = model_class.from_pretrained( + tmpdirname, + torch_dtype=torch.float16, + attn_implementation={"decoder": "sdpa", "audio_encoder": None, "text_encoder": None}, + ) + model.to(torch_device) + + inputs_dict.pop("attention_mask", None) + inputs_dict.pop("decoder_attention_mask", None) + + for name, inp in inputs_dict.items(): + if isinstance(inp, torch.Tensor) and inp.dtype in [torch.float32, torch.float16]: + inputs_dict[name] = inp.to(torch.float16) + + with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False): + _ = model(**inputs_dict) + + @require_flash_attn + @require_torch_gpu + @mark.flash_attn_test + @slow + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence_right_padding def test_flash_attn_2_inference_equivalence_right_padding(self): for model_class in self.all_model_classes: if not model_class._supports_flash_attn_2: @@ -1502,7 +1585,9 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model_fa = model_class.from_pretrained( - tmpdirname, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + tmpdirname, + torch_dtype=torch.bfloat16, + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, ) model_fa.to(torch_device) @@ -1572,7 +1657,7 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_left_padding + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_left_padding def test_flash_attn_2_generate_left_padding(self): # Ignore copy for model_class in self.greedy_sample_model_classes: @@ -1607,7 +1692,7 @@ def test_flash_attn_2_generate_left_padding(self): model = model_class.from_pretrained( tmpdirname, torch_dtype=torch.float16, - attn_implementation="flash_attention_2", + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, low_cpu_mem_usage=True, ).to(torch_device) @@ -1621,7 +1706,7 @@ def test_flash_attn_2_generate_left_padding(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_padding_right + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_padding_right def test_flash_attn_2_generate_padding_right(self): # Ignore copy for model_class in self.greedy_sample_model_classes: @@ -1655,7 +1740,7 @@ def test_flash_attn_2_generate_padding_right(self): model = model_class.from_pretrained( tmpdirname, torch_dtype=torch.float16, - attn_implementation="flash_attention_2", + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, low_cpu_mem_usage=True, ).to(torch_device) @@ -1669,7 +1754,7 @@ def test_flash_attn_2_generate_padding_right(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_use_cache + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_use_cache def test_flash_attn_2_generate_use_cache(self): max_new_tokens = 30 @@ -1698,7 +1783,7 @@ def test_flash_attn_2_generate_use_cache(self): model = model_class.from_pretrained( tmpdirname, torch_dtype=torch.float16, - attn_implementation="flash_attention_2", + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, low_cpu_mem_usage=True, ).to(torch_device) diff --git a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py index 5e13e3d24536..f53fc21ba80c 100644 --- a/tests/models/musicgen_melody/test_modeling_musicgen_melody.py +++ b/tests/models/musicgen_melody/test_modeling_musicgen_melody.py @@ -311,7 +311,9 @@ def test_flash_attn_2_inference_equivalence(self): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model_fa = model_class.from_pretrained( - tmpdirname, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + tmpdirname, + torch_dtype=torch.bfloat16, + attn_implementation="flash_attention_2", ) model_fa.to(torch_device) @@ -391,7 +393,9 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model_fa = model_class.from_pretrained( - tmpdirname, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + tmpdirname, + torch_dtype=torch.bfloat16, + attn_implementation="flash_attention_2", ) model_fa.to(torch_device) @@ -454,144 +458,6 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): assert torch.allclose(logits_fa[:-1], logits[:-1], atol=4e-2, rtol=4e-2) - @require_flash_attn - @require_torch_gpu - @mark.flash_attn_test - @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_left_padding - def test_flash_attn_2_generate_left_padding(self): - # Ignore copy - for model_class in self.greedy_sample_model_classes: - if not model_class._supports_flash_attn_2: - self.skipTest(f"{model_class.__name__} does not support Flash Attention 2") - - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - model = model_class(config) - - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - model = model_class.from_pretrained(tmpdirname, torch_dtype=torch.float16, low_cpu_mem_usage=True).to( - torch_device - ) - - dummy_input = inputs_dict[model.main_input_name] - if dummy_input.dtype in [torch.float32, torch.bfloat16]: - dummy_input = dummy_input.to(torch.float16) - - dummy_attention_mask = inputs_dict.get("attention_mask", torch.ones_like(dummy_input)) - # make sure we do left padding - dummy_attention_mask[:, :-1] = 0 - dummy_attention_mask[:, -1:] = 1 - - out = model.generate( - dummy_input, attention_mask=dummy_attention_mask, max_new_tokens=8, do_sample=False - ) - - model = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch.float16, - attn_implementation="flash_attention_2", - low_cpu_mem_usage=True, - ).to(torch_device) - - out_fa = model.generate( - dummy_input, attention_mask=dummy_attention_mask, max_new_tokens=8, do_sample=False - ) - - self.assertTrue(torch.allclose(out, out_fa)) - - @require_flash_attn - @require_torch_gpu - @mark.flash_attn_test - @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_padding_right - def test_flash_attn_2_generate_padding_right(self): - # Ignore copy - for model_class in self.greedy_sample_model_classes: - if not model_class._supports_flash_attn_2: - self.skipTest(f"{model_class.__name__} does not support Flash Attention 2") - - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - model = model_class(config) - - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - model = model_class.from_pretrained(tmpdirname, torch_dtype=torch.float16, low_cpu_mem_usage=True).to( - torch_device - ) - - dummy_input = inputs_dict[model.main_input_name] - if dummy_input.dtype in [torch.float32, torch.bfloat16]: - dummy_input = dummy_input.to(torch.float16) - - dummy_attention_mask = inputs_dict.get("attention_mask", torch.ones_like(dummy_input)) - # make sure we do right padding - dummy_attention_mask[:, :-1] = 1 - dummy_attention_mask[:, -1:] = 0 - - out = model.generate( - dummy_input, attention_mask=dummy_attention_mask, max_new_tokens=8, do_sample=False - ) - - model = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch.float16, - attn_implementation="flash_attention_2", - low_cpu_mem_usage=True, - ).to(torch_device) - - out_fa = model.generate( - dummy_input, attention_mask=dummy_attention_mask, max_new_tokens=8, do_sample=False - ) - - self.assertTrue(torch.allclose(out, out_fa)) - - @require_flash_attn - @require_torch_gpu - @mark.flash_attn_test - @slow - # Copied from tests.models.musicgen.test_modeling_musicgen.MusicgenDecoderTest.test_flash_attn_2_generate_use_cache - def test_flash_attn_2_generate_use_cache(self): - max_new_tokens = 30 - - # Ignore copy - for model_class in self.greedy_sample_model_classes: - if not model_class._supports_flash_attn_2: - self.skipTest(f"{model_class.__name__} does not support Flash Attention 2") - - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - dummy_input = inputs_dict[model_class.main_input_name] - if dummy_input.dtype in [torch.float32, torch.bfloat16]: - dummy_input = dummy_input.to(torch.float16) - - # make sure that all models have enough positions for generation - if hasattr(config, "max_position_embeddings"): - config.max_position_embeddings = max_new_tokens + dummy_input.shape[1] + 1 - - model = model_class(config) - - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - - dummy_attention_mask = inputs_dict.get("attention_mask", torch.ones_like(dummy_input)) - - model = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch.float16, - attn_implementation="flash_attention_2", - low_cpu_mem_usage=True, - ).to(torch_device) - - # Just test that a large cache works as expected - _ = model.generate( - dummy_input, - attention_mask=dummy_attention_mask, - max_new_tokens=max_new_tokens, - do_sample=False, - use_cache=True, - ) - @parameterized.expand([("float16",), ("bfloat16",), ("float32",)]) @require_torch_sdpa @slow @@ -823,74 +689,6 @@ def get_mean_reldiff(failcase, x, ref, atol, rtol): self.assertTrue(len(fail_cases) == 0, "\n".join(fail_cases)) - @require_torch_sdpa - @slow - # Copied from tests.models.musicgen.test_modeling_musicgen.MusicgenDecoderTest.test_eager_matches_sdpa_generate - def test_eager_matches_sdpa_generate(self): - max_new_tokens = 30 - - # Ignore copy - for model_class in self.greedy_sample_model_classes: - if not model_class._supports_sdpa: - self.skipTest(f"{model_class.__name__} does not support SDPA") - - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - dummy_input = inputs_dict[model_class.main_input_name] - if dummy_input.dtype in [torch.float32, torch.bfloat16]: - dummy_input = dummy_input.to(torch.float16) - - # make sure that all models have enough positions for generation - if hasattr(config, "max_position_embeddings"): - config.max_position_embeddings = max_new_tokens + dummy_input.shape[1] + 1 - - model = model_class(config) - - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - - dummy_attention_mask = inputs_dict.get("attention_mask", torch.ones_like(dummy_input)) - - model_sdpa = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch.float16, - low_cpu_mem_usage=True, - ).to(torch_device) - - self.assertTrue(model_sdpa.config._attn_implementation == "sdpa") - - model_eager = model_class.from_pretrained( - tmpdirname, - torch_dtype=torch.float16, - low_cpu_mem_usage=True, - attn_implementation="eager", - ).to(torch_device) - - self.assertTrue(model_eager.config._attn_implementation == "eager") - - for name, submodule in model_eager.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - raise ValueError("The eager model should not have SDPA attention layers") - - has_sdpa = False - for name, submodule in model_sdpa.named_modules(): - if "SdpaAttention" in submodule.__class__.__name__: - has_sdpa = True - break - if not has_sdpa: - raise ValueError("The SDPA model should have SDPA attention layers") - - # Just test that a large cache works as expected - res_eager = model_eager.generate( - dummy_input, attention_mask=dummy_attention_mask, max_new_tokens=max_new_tokens, do_sample=False - ) - - res_sdpa = model_sdpa.generate( - dummy_input, attention_mask=dummy_attention_mask, max_new_tokens=max_new_tokens, do_sample=False - ) - - self.assertTrue(torch.allclose(res_eager, res_sdpa)) - def prepare_musicgen_melody_inputs_dict( config, @@ -1391,7 +1189,7 @@ def test_save_load_fast_init_from_base(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence def test_flash_attn_2_inference_equivalence(self): for model_class in self.all_model_classes: if not model_class._supports_flash_attn_2: @@ -1403,7 +1201,9 @@ def test_flash_attn_2_inference_equivalence(self): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model_fa = model_class.from_pretrained( - tmpdirname, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + tmpdirname, + torch_dtype=torch.bfloat16, + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, ) model_fa.to(torch_device) @@ -1476,7 +1276,88 @@ def test_flash_attn_2_inference_equivalence(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence_right_padding + def test_flash_attn_2_conversion(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + if not model_class._supports_flash_attn_2: + self.skipTest(f"{model_class.__name__} does not support Flash Attention 2") + + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model = model_class.from_pretrained( + tmpdirname, + torch_dtype=torch.float16, + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, + ).to(torch_device) + + for _, module in model.named_modules(): + if "FlashAttention" in module.__class__.__name__: + return + + self.assertTrue(False, "FlashAttention2 modules not found in model") + + @require_torch_sdpa + @require_torch_gpu + @slow + def test_sdpa_can_dispatch_on_flash(self): + if not self.has_attentions: + self.skipTest(reason="Model architecture does not support attentions") + + torch.compiler.reset() + compute_capability = torch.cuda.get_device_capability() + major, _ = compute_capability + + if not torch.version.cuda or major < 8: + self.skipTest(reason="This test requires an NVIDIA GPU with compute capability >= 8.0") + + for model_class in self.all_model_classes: + if not model_class._supports_sdpa: + self.skipTest(f"{model_class.__name__} does not support SDPA") + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + inputs_dict = self._prepare_for_class(inputs_dict, model_class) + if config.model_type in ["llava", "llava_next", "vipllava", "video_llava"]: + self.skipTest( + reason="Llava-like models currently (transformers==4.39.1) requires an attention_mask input" + ) + if config.model_type in ["paligemma"]: + self.skipTest( + "PaliGemma-like models currently (transformers==4.41.0) requires an attention_mask input" + ) + if config.model_type in ["idefics", "idefics2", "idefics3"]: + self.skipTest(reason="Idefics currently (transformers==4.39.1) requires an image_attention_mask input") + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model = model_class.from_pretrained( + tmpdirname, + torch_dtype=torch.float16, + attn_implementation={"decoder": "sdpa", "audio_encoder": None, "text_encoder": None}, + ) + model.to(torch_device) + + inputs_dict.pop("attention_mask", None) + inputs_dict.pop("decoder_attention_mask", None) + + for name, inp in inputs_dict.items(): + if isinstance(inp, torch.Tensor) and inp.dtype in [torch.float32, torch.float16]: + inputs_dict[name] = inp.to(torch.float16) + + with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False): + _ = model(**inputs_dict) + + @require_flash_attn + @require_torch_gpu + @mark.flash_attn_test + @slow + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_inference_equivalence_right_padding def test_flash_attn_2_inference_equivalence_right_padding(self): for model_class in self.all_model_classes: if not model_class._supports_flash_attn_2: @@ -1488,7 +1369,9 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): with tempfile.TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model_fa = model_class.from_pretrained( - tmpdirname, torch_dtype=torch.bfloat16, attn_implementation="flash_attention_2" + tmpdirname, + torch_dtype=torch.bfloat16, + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, ) model_fa.to(torch_device) @@ -1558,7 +1441,7 @@ def test_flash_attn_2_inference_equivalence_right_padding(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_left_padding + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_left_padding def test_flash_attn_2_generate_left_padding(self): # Ignore copy for model_class in self.greedy_sample_model_classes: @@ -1593,7 +1476,7 @@ def test_flash_attn_2_generate_left_padding(self): model = model_class.from_pretrained( tmpdirname, torch_dtype=torch.float16, - attn_implementation="flash_attention_2", + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, low_cpu_mem_usage=True, ).to(torch_device) @@ -1607,7 +1490,7 @@ def test_flash_attn_2_generate_left_padding(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_padding_right + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_padding_right def test_flash_attn_2_generate_padding_right(self): # Ignore copy for model_class in self.greedy_sample_model_classes: @@ -1641,7 +1524,7 @@ def test_flash_attn_2_generate_padding_right(self): model = model_class.from_pretrained( tmpdirname, torch_dtype=torch.float16, - attn_implementation="flash_attention_2", + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, low_cpu_mem_usage=True, ).to(torch_device) @@ -1655,7 +1538,7 @@ def test_flash_attn_2_generate_padding_right(self): @require_torch_gpu @mark.flash_attn_test @slow - # Copied from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_use_cache + # Adapted from tests.test_modeling_common.ModelTesterMixin.test_flash_attn_2_generate_use_cache def test_flash_attn_2_generate_use_cache(self): max_new_tokens = 30 @@ -1684,7 +1567,7 @@ def test_flash_attn_2_generate_use_cache(self): model = model_class.from_pretrained( tmpdirname, torch_dtype=torch.float16, - attn_implementation="flash_attention_2", + attn_implementation={"decoder": "flash_attention_2", "audio_encoder": None, "text_encoder": None}, low_cpu_mem_usage=True, ).to(torch_device) diff --git a/tests/models/paligemma/test_modeling_paligemma.py b/tests/models/paligemma/test_modeling_paligemma.py index d8368b3e4434..cfc2a2c29b1d 100644 --- a/tests/models/paligemma/test_modeling_paligemma.py +++ b/tests/models/paligemma/test_modeling_paligemma.py @@ -320,6 +320,16 @@ def test_generate_from_inputs_embeds_with_static_cache(self): def test_static_cache_matches_dynamic(self): pass + @unittest.skip("FlashAttention only support fp16 and bf16 data type") + def test_flash_attn_2_fp32_ln(self): + pass + + @unittest.skip( + "VLMs need lots of steps to prepare images/mask correctly to get pad-free inputs. Can be tested as part of LLM test" + ) + def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): + pass + @slow @require_torch diff --git a/tests/models/video_llava/test_modeling_video_llava.py b/tests/models/video_llava/test_modeling_video_llava.py index b3be07060161..1bd01843981d 100644 --- a/tests/models/video_llava/test_modeling_video_llava.py +++ b/tests/models/video_llava/test_modeling_video_llava.py @@ -238,6 +238,16 @@ def test_sdpa_can_compile_dynamic(self): def test_sdpa_can_dispatch_on_flash(self): pass + @unittest.skip("FlashAttention only support fp16 and bf16 data type") + def test_flash_attn_2_fp32_ln(self): + pass + + @unittest.skip( + "VLMs need lots of steps to prepare images/mask correctly to get pad-free inputs. Can be tested as part of LLM test" + ) + def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): + pass + @unittest.skip( reason="After #33533, this still passes, but many subsequential tests fail with `device-side assert triggered`" ) diff --git a/tests/models/vipllava/test_modeling_vipllava.py b/tests/models/vipllava/test_modeling_vipllava.py index ea5dab4176d6..2c241c23f261 100644 --- a/tests/models/vipllava/test_modeling_vipllava.py +++ b/tests/models/vipllava/test_modeling_vipllava.py @@ -243,6 +243,16 @@ def test_sdpa_can_compile_dynamic(self): def test_sdpa_can_dispatch_on_flash(self): pass + @unittest.skip("FlashAttention only support fp16 and bf16 data type") + def test_flash_attn_2_fp32_ln(self): + pass + + @unittest.skip( + "VLMs need lots of steps to prepare images/mask correctly to get pad-free inputs. Can be tested as part of LLM test" + ) + def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): + pass + @require_torch class VipLlavaForConditionalGenerationIntegrationTest(unittest.TestCase): diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index ea79db461809..dec1482f562a 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4403,7 +4403,7 @@ def test_sdpa_can_dispatch_on_flash(self): self.skipTest( "PaliGemma-like models currently (transformers==4.41.0) requires an attention_mask input" ) - if config.model_type in ["idefics"]: + if config.model_type in ["idefics", "idefics2", "idefics3"]: self.skipTest(reason="Idefics currently (transformers==4.39.1) requires an image_attention_mask input") model = model_class(config) @@ -4843,7 +4843,7 @@ def test_flash_attention_2_padding_matches_padding_free_with_position_ids(self): if 0 in inputs_dict["attention_mask"][:, -1]: inputs_dict["attention_mask"] = inputs_dict["attention_mask"].flip(1) dummy_attention_mask = inputs_dict["attention_mask"] - inputs_dict["input_ids"][~dummy_attention_mask.bool()] = config.pad_token_id + inputs_dict["input_ids"][~dummy_attention_mask.bool()] = config.get_text_config().pad_token_id model = ( model_class.from_pretrained(