From bad26e409ca84de8904b042c7c7c3687c4f91671 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 2 Aug 2023 13:25:54 +0000 Subject: [PATCH 1/6] [MMS] Fix mms --- src/transformers/modeling_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 7c39733be4f2..05d7ad505b47 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2930,6 +2930,7 @@ def from_pretrained( if dtype_orig is not None: torch.set_default_dtype(dtype_orig) + model.tie_weights() ( model, missing_keys, From 153af5c75135622910949e3b598ee9dd93dbf03c Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 2 Aug 2023 13:48:28 +0000 Subject: [PATCH 2/6] [MMS] Fix mms --- src/transformers/modeling_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 05d7ad505b47..cc1d447b468b 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2930,7 +2930,6 @@ def from_pretrained( if dtype_orig is not None: torch.set_default_dtype(dtype_orig) - model.tie_weights() ( model, missing_keys, @@ -3049,6 +3048,7 @@ def _load_pretrained_model( is_sharded_safetensors = is_safetensors and sharded_metadata is not None # Retrieve missing & unexpected_keys + model.tie_weights() model_state_dict = model.state_dict() expected_keys = list(model_state_dict.keys()) prefix = model.base_model_prefix @@ -3224,6 +3224,7 @@ def _find_mismatched_keys( folder = os.path.sep.join(resolved_archive_file[0].split(os.path.sep)[:-1]) else: folder = None + if device_map is not None and is_safetensors: param_device_map = expand_device_map(device_map, original_loaded_keys) @@ -3281,6 +3282,7 @@ def _find_mismatched_keys( if len(resolved_archive_file) > 1: resolved_archive_file = logging.tqdm(resolved_archive_file, desc="Loading checkpoint shards") + for shard_file in resolved_archive_file: # Skip the load for shards that only contain disk-offloaded weights when using safetensors for the offload. if shard_file in disk_only_shard_files: From 35cf772ae8d24190bbefb58ecf5ece517ce9ef99 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 2 Aug 2023 15:04:43 +0000 Subject: [PATCH 3/6] fix mms loading --- src/transformers/modeling_utils.py | 7 ++-- .../models/wav2vec2/test_modeling_wav2vec2.py | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index cc1d447b468b..ae4b4fac26b1 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3047,10 +3047,14 @@ def _load_pretrained_model( offload_state_dict = True is_sharded_safetensors = is_safetensors and sharded_metadata is not None - # Retrieve missing & unexpected_keys + + # tie the model weights before retrieving the state_dict model.tie_weights() + + # Retrieve missing & unexpected_keys model_state_dict = model.state_dict() expected_keys = list(model_state_dict.keys()) + prefix = model.base_model_prefix def _fix_key(key): @@ -3093,7 +3097,6 @@ def _fix_key(key): model_buffers = {".".join([prefix, key]) for key in model_buffers} unexpected_keys = list(unexpected_keys - model_buffers) - model.tie_weights() if device_map is None: ptrs = collections.defaultdict(list) for name, tensor in model.state_dict().items(): diff --git a/tests/models/wav2vec2/test_modeling_wav2vec2.py b/tests/models/wav2vec2/test_modeling_wav2vec2.py index 4db9b156db46..252b30689df5 100644 --- a/tests/models/wav2vec2/test_modeling_wav2vec2.py +++ b/tests/models/wav2vec2/test_modeling_wav2vec2.py @@ -1151,6 +1151,40 @@ def get_logits(model, input_features): self.assertTrue(torch.allclose(logits, logits_2, atol=1e-3)) + def test_load_and_new_target_lang_adapter(self): + processor = Wav2Vec2Processor.from_pretrained( + "hf-internal-testing/tiny-random-wav2vec2", return_attention_mask=True + ) + + def get_logits(model, input_features): + model = model.to(torch_device) + batch = processor( + input_features, + padding=True, + sampling_rate=processor.feature_extractor.sampling_rate, + return_tensors="pt", + ) + + with torch.no_grad(): + logits = model( + input_values=batch["input_values"].to(torch_device), + attention_mask=batch["attention_mask"].to(torch_device), + ).logits + return logits + + input_features = [np.random.random(16_000 * s) for s in [1, 3, 2, 6]] + + model = Wav2Vec2ForCTC.from_pretrained("hf-internal-testing/tiny-random-wav2vec2-adapter", target_lang="fr", ignore_mismatched_sizes=True) + + logits = get_logits(model, input_features) + + model_2 = Wav2Vec2ForCTC.from_pretrained("hf-internal-testing/tiny-random-wav2vec2-adapter") + model_2.load_adapter("fr") + + logits_2 = get_logits(model_2, input_features) + + self.assertTrue(torch.allclose(logits, logits_2, atol=1e-3)) + def test_load_attn_adapter(self): processor = Wav2Vec2Processor.from_pretrained( "hf-internal-testing/tiny-random-wav2vec2", return_attention_mask=True From 3d97b19674ff72d91f86a8856395c5b1b41fed25 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 2 Aug 2023 17:11:59 +0200 Subject: [PATCH 4/6] Apply suggestions from code review --- src/transformers/modeling_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index ae4b4fac26b1..9e167cfdeeb4 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3054,7 +3054,6 @@ def _load_pretrained_model( # Retrieve missing & unexpected_keys model_state_dict = model.state_dict() expected_keys = list(model_state_dict.keys()) - prefix = model.base_model_prefix def _fix_key(key): @@ -3227,7 +3226,6 @@ def _find_mismatched_keys( folder = os.path.sep.join(resolved_archive_file[0].split(os.path.sep)[:-1]) else: folder = None - if device_map is not None and is_safetensors: param_device_map = expand_device_map(device_map, original_loaded_keys) @@ -3285,7 +3283,6 @@ def _find_mismatched_keys( if len(resolved_archive_file) > 1: resolved_archive_file = logging.tqdm(resolved_archive_file, desc="Loading checkpoint shards") - for shard_file in resolved_archive_file: # Skip the load for shards that only contain disk-offloaded weights when using safetensors for the offload. if shard_file in disk_only_shard_files: From 0ccd919932c8e89867ab14be40f9990545bd2e2b Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 2 Aug 2023 15:25:13 +0000 Subject: [PATCH 5/6] make style --- tests/models/wav2vec2/test_modeling_wav2vec2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/models/wav2vec2/test_modeling_wav2vec2.py b/tests/models/wav2vec2/test_modeling_wav2vec2.py index 252b30689df5..b868fa984bce 100644 --- a/tests/models/wav2vec2/test_modeling_wav2vec2.py +++ b/tests/models/wav2vec2/test_modeling_wav2vec2.py @@ -1174,7 +1174,9 @@ def get_logits(model, input_features): input_features = [np.random.random(16_000 * s) for s in [1, 3, 2, 6]] - model = Wav2Vec2ForCTC.from_pretrained("hf-internal-testing/tiny-random-wav2vec2-adapter", target_lang="fr", ignore_mismatched_sizes=True) + model = Wav2Vec2ForCTC.from_pretrained( + "hf-internal-testing/tiny-random-wav2vec2-adapter", target_lang="fr", ignore_mismatched_sizes=True + ) logits = get_logits(model, input_features) From e73a0590204d68c66e58713a394a0b21b5b45873 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 2 Aug 2023 17:50:41 +0200 Subject: [PATCH 6/6] Update tests/models/wav2vec2/test_modeling_wav2vec2.py --- tests/models/wav2vec2/test_modeling_wav2vec2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/models/wav2vec2/test_modeling_wav2vec2.py b/tests/models/wav2vec2/test_modeling_wav2vec2.py index b868fa984bce..630a5d8e8513 100644 --- a/tests/models/wav2vec2/test_modeling_wav2vec2.py +++ b/tests/models/wav2vec2/test_modeling_wav2vec2.py @@ -1151,7 +1151,8 @@ def get_logits(model, input_features): self.assertTrue(torch.allclose(logits, logits_2, atol=1e-3)) - def test_load_and_new_target_lang_adapter(self): + # test that loading adapter weights with mismatched vocab sizes can be loaded + def test_load_target_lang_with_mismatched_size(self): processor = Wav2Vec2Processor.from_pretrained( "hf-internal-testing/tiny-random-wav2vec2", return_attention_mask=True )