From 6de9173f18ffb919331f77fea6e0a55e40791c0b Mon Sep 17 00:00:00 2001 From: raushan Date: Thu, 31 Oct 2024 12:25:05 +0100 Subject: [PATCH 1/7] fix --- src/transformers/modeling_utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 8481fa7df9cd..38c1eb00f511 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1422,9 +1422,7 @@ def __init__(self, config: PretrainedConfig, *inputs, **kwargs): ) # Save config and origin of the pretrained weights if given in model if not getattr(config, "_attn_implementation_autoset", False): - config = self._autoset_attn_implementation( - config, torch_dtype=torch.get_default_dtype(), check_device_map=False - ) + config = self._autoset_attn_implementation(config, torch_dtype=config.torch_dtype, check_device_map=False) self.config = config self.name_or_path = config.name_or_path @@ -1505,7 +1503,7 @@ def _from_config(cls, config, **kwargs): # when we init a model from within another model (e.g. VLMs) and dispatch on FA2 # a warning is raised that dtype should be fp16. Since we never pass dtype from within # modeling code, we can try to infer it here same way as done in `from_pretrained` - torch_dtype = kwargs.pop("torch_dtype", torch.get_default_dtype()) + torch_dtype = kwargs.pop("torch_dtype", config.torch_dtype) use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False) # override default dtype if needed @@ -4047,6 +4045,14 @@ def from_pretrained( ) elif hasattr(torch, torch_dtype): torch_dtype = getattr(torch, torch_dtype) + for sub_config_key in config.sub_configs.keys(): + sub_config = getattr(config, sub_config_key) + sub_config.torch_dtype = torch_dtype + elif isinstance(torch_dtype, dict): + for key, curr_dtype in torch_dtype.items(): + if hasattr(config, key): + value = getattr(config, key) + value.torch_dtype = curr_dtype else: raise ValueError( f'`torch_dtype` can be one of: `torch.dtype`, `"auto"` or a string of a valid `torch.dtype`, but received {torch_dtype}' From 54cc1652b65eb833e72906d8a466822e4d893fc3 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 22 Nov 2024 08:54:30 +0100 Subject: [PATCH 2/7] fix test --- src/transformers/modeling_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 38c1eb00f511..55f3e425a75b 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1504,6 +1504,9 @@ def _from_config(cls, config, **kwargs): # a warning is raised that dtype should be fp16. Since we never pass dtype from within # modeling code, we can try to infer it here same way as done in `from_pretrained` torch_dtype = kwargs.pop("torch_dtype", config.torch_dtype) + if isinstance(torch_dtype, str): + torch_dtype = getattr(torch, torch_dtype) + use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False) # override default dtype if needed From 8f07e893fbc3853984a8d480bd65bdf85c2108b2 Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 26 Nov 2024 09:51:54 +0100 Subject: [PATCH 3/7] add tests --- src/transformers/modeling_utils.py | 27 ++++++++++----- .../models/qwen2_vl/test_modeling_qwen2_vl.py | 1 + tests/utils/test_modeling_utils.py | 34 +++++++++++++++++++ 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6636da6e26ec..b4c7b00ee66e 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4017,15 +4017,24 @@ def from_pretrained( for sub_config_key in config.sub_configs.keys(): sub_config = getattr(config, sub_config_key) sub_config.torch_dtype = torch_dtype - elif isinstance(torch_dtype, dict): - for key, curr_dtype in torch_dtype.items(): - if hasattr(config, key): - value = getattr(config, key) - value.torch_dtype = curr_dtype - else: - raise ValueError( - f'`torch_dtype` can be one of: `torch.dtype`, `"auto"` or a string of a valid `torch.dtype`, but received {torch_dtype}' - ) + elif isinstance(torch_dtype, torch.dtype): + pass + elif isinstance(torch_dtype, dict): + for key, curr_dtype in torch_dtype.items(): + if hasattr(config, key): + value = getattr(config, key) + value.torch_dtype = curr_dtype + # the main dtype by default will be the text model's dtype + torch_dtype = config.get_text_config().torch_dtype + if hasattr(torch, torch_dtype): + torch_dtype = getattr(torch, torch_dtype) + elif torch_dtype is None: + torch_dtype = torch.float32 + else: + raise ValueError( + f'`torch_dtype` can be one of: `torch.dtype`, `"auto"`, a string of a valid `torch.dtype` or ' + f"a `dict` with valid `torch_dtype` for each sub-config in composite configs, but received {torch_dtype}" + ) dtype_orig = cls._set_default_torch_dtype(torch_dtype) # Check if `_keep_in_fp32_modules` is not None diff --git a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py index 93ed33ae7744..e1f723f53c1a 100644 --- a/tests/models/qwen2_vl/test_modeling_qwen2_vl.py +++ b/tests/models/qwen2_vl/test_modeling_qwen2_vl.py @@ -229,6 +229,7 @@ class Qwen2VLModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCas pipeline_model_mapping = {"image-text-to-text": Qwen2VLForConditionalGeneration} test_pruning = False test_head_masking = False + _is_composite = True def setUp(self): self.model_tester = Qwen2VLVisionText2TextModelTester(self) diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index 85e7c20dd527..abe526726a69 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -37,6 +37,7 @@ AutoModel, AutoModelForImageClassification, AutoModelForSequenceClassification, + LlavaForConditionalGeneration, OwlViTForObjectDetection, PretrainedConfig, is_torch_available, @@ -300,6 +301,7 @@ def test_local_files_only(self): TINY_BERT_FOR_TOKEN_CLASSIFICATION = "hf-internal-testing/tiny-bert-for-token-classification" TINY_MISTRAL = "hf-internal-testing/tiny-random-MistralForCausalLM" TINY_IMAGE_CLASSIF = "hf-internal-testing/tiny-random-SiglipForImageClassification" +TINY_LLAVA = "hf-internal-testing/tiny-random-LlavaForConditionalGeneration" LOG = logging.get_logger(__name__) @@ -460,6 +462,38 @@ def test_model_from_config_torch_dtype_str(self): with self.assertRaises(ValueError): model = AutoModel.from_pretrained(TINY_T5, torch_dtype="int64") + def test_model_from_config_torch_dtype_composite(self): + """ + Test that from_pretrained works with torch_dtype being as a dict per each sub-config in composite config + """ + model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, torch_dtype="float32") + self.assertEqual(model.language_model.dtype, torch.float32) + self.assertEqual(model.vision_tower.dtype, torch.float32) + + model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, torch_dtype="float16") + self.assertEqual(model.language_model.dtype, torch.float16) + self.assertEqual(model.vision_tower.dtype, torch.float16) + + model = LlavaForConditionalGeneration.from_pretrained( + TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "float16"} + ) + self.assertEqual(model.language_model.dtype, torch.float32) + self.assertEqual(model.vision_tower.dtype, torch.float16) + + config = copy.deepcopy(model.config) + config.torch_dtype = torch.float32 + config.vision_config.torch_dtype = "float16" + model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, config=config, torch_dtype="auto") + self.assertEqual(model.language_model.dtype, torch.float32) + self.assertEqual(model.vision_tower.dtype, torch.float16) + + # torch.set_default_dtype() supports only float dtypes, so will fail with non-float type + with self.assertRaises(ValueError): + model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, torch_dtype="int64") + model = LlavaForConditionalGeneration.from_pretrained( + TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "int64"} + ) + @require_torch def test_model_from_pretrained_meta_device(self): def is_on_meta(model_id, dtype): From e34f6afd79f487f269a48d699c5554740e5fa2e6 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 29 Nov 2024 17:05:40 +0100 Subject: [PATCH 4/7] add more tests --- src/transformers/configuration_utils.py | 7 +++-- src/transformers/modeling_utils.py | 12 ++++--- .../models/chameleon/modeling_chameleon.py | 2 +- tests/utils/test_modeling_utils.py | 31 ++++++++++++++++--- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py index e49eab86b4e1..2d0ba7879ee4 100755 --- a/src/transformers/configuration_utils.py +++ b/src/transformers/configuration_utils.py @@ -988,8 +988,11 @@ def dict_torch_dtype_to_str(self, d: Dict[str, Any]) -> None: converts torch.dtype to a string of just the type. For example, `torch.float32` get converted into *"float32"* string, which can then be stored in the json format. """ - if d.get("torch_dtype", None) is not None and not isinstance(d["torch_dtype"], str): - d["torch_dtype"] = str(d["torch_dtype"]).split(".")[1] + if d.get("torch_dtype", None) is not None: + if isinstance(d["torch_dtype"], dict): + d["torch_dtype"] = {k: str(v).split(".")[-1] for k, v in d["torch_dtype"].items()} + elif not isinstance(d["torch_dtype"], str): + d["torch_dtype"] = str(d["torch_dtype"]).split(".")[1] for value in d.values(): if isinstance(value, dict): self.dict_torch_dtype_to_str(value) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index b4c7b00ee66e..ef4d6667fc84 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4024,17 +4024,19 @@ def from_pretrained( if hasattr(config, key): value = getattr(config, key) value.torch_dtype = curr_dtype - # the main dtype by default will be the text model's dtype - torch_dtype = config.get_text_config().torch_dtype - if hasattr(torch, torch_dtype): + # main torch dtype for modules that aren't part of any sub-config + torch_dtype = torch_dtype.get("") + config.torch_dtype = torch_dtype + if isinstance(torch_dtype, str) and hasattr(torch, torch_dtype): torch_dtype = getattr(torch, torch_dtype) elif torch_dtype is None: torch_dtype = torch.float32 else: raise ValueError( - f'`torch_dtype` can be one of: `torch.dtype`, `"auto"`, a string of a valid `torch.dtype` or ' - f"a `dict` with valid `torch_dtype` for each sub-config in composite configs, but received {torch_dtype}" + f"`torch_dtype` can be one of: `torch.dtype`, `'auto'`, a string of a valid `torch.dtype` or a `dict` with valid `torch_dtype` " + f"for each sub-config in composite configs, but received {torch_dtype}" ) + dtype_orig = cls._set_default_torch_dtype(torch_dtype) # Check if `_keep_in_fp32_modules` is not None diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 3255b6f44c05..79452849c074 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -1215,7 +1215,7 @@ def __init__(self, config: ChameleonConfig): [decoder_layer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] ) self.norm = ChameleonRMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.vqmodel = ChameleonVQVAE(config.vq_config) + self.vqmodel = ChameleonVQVAE._from_config(config.vq_config) self.gradient_checkpointing = False # Initialize weights and apply final processing diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index abe526726a69..24595c20054c 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -466,6 +466,7 @@ def test_model_from_config_torch_dtype_composite(self): """ Test that from_pretrained works with torch_dtype being as a dict per each sub-config in composite config """ + # should be able to set torch_dtype as a simple string and the model loads it correctly model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, torch_dtype="float32") self.assertEqual(model.language_model.dtype, torch.float32) self.assertEqual(model.vision_tower.dtype, torch.float32) @@ -474,24 +475,44 @@ def test_model_from_config_torch_dtype_composite(self): self.assertEqual(model.language_model.dtype, torch.float16) self.assertEqual(model.vision_tower.dtype, torch.float16) + # should be able to set torch_dtype as a dict for each sub-config model = LlavaForConditionalGeneration.from_pretrained( - TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "float16"} + TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "float16", "": "bfloat16"} ) self.assertEqual(model.language_model.dtype, torch.float32) self.assertEqual(model.vision_tower.dtype, torch.float16) + self.assertEqual(model.multi_modal_projector.linear_1.weight.dtype, torch.bfloat16) + # should be able to set the values as torch.dtype + model = LlavaForConditionalGeneration.from_pretrained( + TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "float16", "": "bfloat16"} + ) + self.assertEqual(model.language_model.dtype, torch.float32) + self.assertEqual(model.vision_tower.dtype, torch.float16) + self.assertEqual(model.multi_modal_projector.linear_1.weight.dtype, torch.bfloat16) + + # should be able to set the values in configs directly and pass it to `from_pretrained` config = copy.deepcopy(model.config) - config.torch_dtype = torch.float32 - config.vision_config.torch_dtype = "float16" + config.text_config.torch_dtype = torch.float32 + config.vision_config.torch_dtype = torch.bfloat16 + config.torch_dtype = torch.float16 model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, config=config, torch_dtype="auto") self.assertEqual(model.language_model.dtype, torch.float32) - self.assertEqual(model.vision_tower.dtype, torch.float16) + self.assertEqual(model.vision_tower.dtype, torch.bfloat16) + self.assertEqual(model.multi_modal_projector.linear_1.weight.dtype, torch.float16) + + # but if the model has `_keep_in_fp32_modules` then those modules should be in fp32 no matter what + LlavaForConditionalGeneration._keep_in_fp32_modules = ["multi_modal_projector"] + model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, config=config, torch_dtype="auto") + self.assertEqual(model.language_model.dtype, torch.float32) + self.assertEqual(model.vision_tower.dtype, torch.bfloat16) + self.assertEqual(model.multi_modal_projector.linear_1.weight.dtype, torch.float32) # torch.set_default_dtype() supports only float dtypes, so will fail with non-float type with self.assertRaises(ValueError): model = LlavaForConditionalGeneration.from_pretrained(TINY_LLAVA, torch_dtype="int64") model = LlavaForConditionalGeneration.from_pretrained( - TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "int64"} + TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "int64", "": "float16"} ) @require_torch From 0fdafb7c3436d6aced2d683599888c02dbd91114 Mon Sep 17 00:00:00 2001 From: raushan Date: Fri, 29 Nov 2024 17:27:08 +0100 Subject: [PATCH 5/7] fix tests --- src/transformers/modeling_utils.py | 5 +- .../models/chameleon/modeling_chameleon.py | 112 +++++++++--------- 2 files changed, 59 insertions(+), 58 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index ef4d6667fc84..c7d73d657880 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1373,9 +1373,10 @@ def __init__(self, config: PretrainedConfig, *inputs, **kwargs): "`PretrainedConfig`. To create a model from a pretrained model use " f"`model = {self.__class__.__name__}.from_pretrained(PRETRAINED_MODEL_NAME)`" ) - # Save config and origin of the pretrained weights if given in model if not getattr(config, "_attn_implementation_autoset", False): - config = self._autoset_attn_implementation(config, torch_dtype=config.torch_dtype, check_device_map=False) + # config usually has a `torch_dtype` but we need the next line for the `no_super_init` tests + dtype = config.torch_dtype if hasattr(config, "torch_dtype") else torch.get_default_dtype() + config = self._autoset_attn_implementation(config, torch_dtype=dtype, check_device_map=False) self.config = config self.name_or_path = config.name_or_path diff --git a/src/transformers/models/chameleon/modeling_chameleon.py b/src/transformers/models/chameleon/modeling_chameleon.py index 79452849c074..db42edb82812 100644 --- a/src/transformers/models/chameleon/modeling_chameleon.py +++ b/src/transformers/models/chameleon/modeling_chameleon.py @@ -971,62 +971,6 @@ def forward(self, pixel_values: torch.LongTensor): return last_hidden_state -CHAMELEON_VQ_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`ChameleonVQVAEConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - """The VQ-VAE model used in Chameleon for encoding/decoding images into discrete tokens. - This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from - [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv Taigman](https://arxiv.org/abs/2203.13131). - """, - CHAMELEON_VQ_START_DOCSTRING, -) -class ChameleonVQVAE(PreTrainedModel): - config_class = ChameleonVQVAEConfig - _no_split_modules = ["ChameleonVQVAEVectorQuantizer"] - - def _init_weights(self, module): - std = self.config.initializer_range - if isinstance(module, nn.Embedding): - module.weight.data.normal_(mean=0.0, std=std) - elif isinstance(module, nn.GroupNorm): - module.bias.data.zero_() - module.weight.data.fill_(1.0) - elif isinstance(module, (nn.Linear, nn.Conv2d)): - module.weight.data.normal_(mean=0.0, std=std) - if module.bias is not None: - module.bias.data.zero_() - - def __init__(self, config: ChameleonVQVAEConfig): - super().__init__(config) - - self.encoder = ChameleonVQVAEEncoder(config) - self.quantize = ChameleonVQVAEVectorQuantizer(config) - self.quant_conv = torch.nn.Conv2d(config.latent_channels, config.embed_dim, 1) - self.post_quant_conv = torch.nn.Conv2d(config.embed_dim, config.latent_channels, 1) - self.eval() # Chameleon's VQ model is frozen - - def encode(self, pixel_values: torch.LongTensor): - hidden_states = self.encoder(pixel_values) - hidden_states = self.quant_conv(hidden_states) - quant, emb_loss, indices = self.quantize(hidden_states) - return quant, emb_loss, indices - - class ChameleonImageVocabularyMapping: """ A class for mapping discrete image tokens from VQGAN to BPE tokens. @@ -1122,6 +1066,62 @@ def _init_weights(self, module): module.weight.data[module.padding_idx].zero_() +CHAMELEON_VQ_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`ChameleonVQVAEConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + """The VQ-VAE model used in Chameleon for encoding/decoding images into discrete tokens. + This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from + [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv Taigman](https://arxiv.org/abs/2203.13131). + """, + CHAMELEON_VQ_START_DOCSTRING, +) +class ChameleonVQVAE(ChameleonPreTrainedModel): + config_class = ChameleonVQVAEConfig + _no_split_modules = ["ChameleonVQVAEVectorQuantizer"] + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + elif isinstance(module, nn.GroupNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + elif isinstance(module, (nn.Linear, nn.Conv2d)): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + + def __init__(self, config: ChameleonVQVAEConfig): + super().__init__(config) + + self.encoder = ChameleonVQVAEEncoder(config) + self.quantize = ChameleonVQVAEVectorQuantizer(config) + self.quant_conv = torch.nn.Conv2d(config.latent_channels, config.embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d(config.embed_dim, config.latent_channels, 1) + self.eval() # Chameleon's VQ model is frozen + + def encode(self, pixel_values: torch.LongTensor): + hidden_states = self.encoder(pixel_values) + hidden_states = self.quant_conv(hidden_states) + quant, emb_loss, indices = self.quantize(hidden_states) + return quant, emb_loss, indices + + CHAMELEON_INPUTS_DOCSTRING = r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): From b4bf4be87cf66eb1bb98a6af13db174739412f16 Mon Sep 17 00:00:00 2001 From: raushan Date: Tue, 10 Dec 2024 12:04:58 +0100 Subject: [PATCH 6/7] supposed to be a torch.dtype test --- tests/utils/test_modeling_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index 24595c20054c..078c88eb8339 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -483,9 +483,9 @@ def test_model_from_config_torch_dtype_composite(self): self.assertEqual(model.vision_tower.dtype, torch.float16) self.assertEqual(model.multi_modal_projector.linear_1.weight.dtype, torch.bfloat16) - # should be able to set the values as torch.dtype + # should be able to set the values as torch.dtype (not str) model = LlavaForConditionalGeneration.from_pretrained( - TINY_LLAVA, torch_dtype={"text_config": "float32", "vision_config": "float16", "": "bfloat16"} + TINY_LLAVA, torch_dtype={"text_config": torch.float32, "vision_config": torch.float16, "": torch.bfloat16} ) self.assertEqual(model.language_model.dtype, torch.float32) self.assertEqual(model.vision_tower.dtype, torch.float16) From 4a590740da451706a6d055751a0e9a731cd6874d Mon Sep 17 00:00:00 2001 From: raushan Date: Wed, 8 Jan 2025 17:51:36 +0100 Subject: [PATCH 7/7] handle BC and make fp32 default --- src/transformers/modeling_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index c7d73d657880..4d2c32550158 100755 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4039,6 +4039,13 @@ def from_pretrained( ) dtype_orig = cls._set_default_torch_dtype(torch_dtype) + else: + # set fp32 as the default dtype for BC + default_dtype = str(torch.get_default_dtype()).split(".")[-1] + config.torch_dtype = default_dtype + for key in config.sub_configs.keys(): + value = getattr(config, key) + value.torch_dtype = default_dtype # Check if `_keep_in_fp32_modules` is not None use_keep_in_fp32_modules = (cls._keep_in_fp32_modules is not None) and (