From 56b9b2cb584541b2d65016908709bd12e8558eb2 Mon Sep 17 00:00:00 2001 From: Johannes Kolbe Date: Mon, 28 Mar 2022 21:54:56 +0200 Subject: [PATCH 01/54] add doctests to TF ViT --- src/transformers/models/vit/modeling_tf_vit.py | 5 +++-- utils/documentation_tests.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py index e2e946d8c9f4..36d3bbc69a07 100644 --- a/src/transformers/models/vit/modeling_tf_vit.py +++ b/src/transformers/models/vit/modeling_tf_vit.py @@ -662,7 +662,6 @@ def call( Returns: Examples: - ```python >>> from transformers import ViTFeatureExtractor, TFViTModel >>> from PIL import Image @@ -677,6 +676,8 @@ def call( >>> inputs = feature_extractor(images=image, return_tensors="tf") >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + [1, 197, 768] ```""" outputs = self.vit( @@ -768,7 +769,6 @@ def call( Returns: Examples: - ```python >>> from transformers import ViTFeatureExtractor, TFViTForImageClassification >>> import tensorflow as tf @@ -787,6 +787,7 @@ def call( >>> # model predicts one of the 1000 ImageNet classes >>> predicted_class_idx = tf.math.argmax(logits, axis=-1)[0] >>> print("Predicted class:", model.config.id2label[int(predicted_class_idx)]) + Predicted class: Egyptian cat ```""" outputs = self.vit( diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index b5d9f8570cbc..3983af8e7fb3 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -44,6 +44,7 @@ src/transformers/models/van/modeling_van.py src/transformers/models/vilt/modeling_vilt.py src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py src/transformers/models/vit/modeling_vit.py +src/transformers/models/vit/modeling_tf_vit.py src/transformers/models/vit_mae/modeling_vit_mae.py src/transformers/models/wav2vec2/modeling_wav2vec2.py src/transformers/models/wav2vec2/tokenization_wav2vec2.py From fbd04612a87c32deadff9cbf76dc7b0c01806e5f Mon Sep 17 00:00:00 2001 From: Johannes Kolbe Date: Mon, 28 Mar 2022 21:54:56 +0200 Subject: [PATCH 02/54] add doctests to TF ViT --- src/transformers/models/vit/modeling_tf_vit.py | 5 +++-- utils/documentation_tests.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py index e2e946d8c9f4..36d3bbc69a07 100644 --- a/src/transformers/models/vit/modeling_tf_vit.py +++ b/src/transformers/models/vit/modeling_tf_vit.py @@ -662,7 +662,6 @@ def call( Returns: Examples: - ```python >>> from transformers import ViTFeatureExtractor, TFViTModel >>> from PIL import Image @@ -677,6 +676,8 @@ def call( >>> inputs = feature_extractor(images=image, return_tensors="tf") >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + [1, 197, 768] ```""" outputs = self.vit( @@ -768,7 +769,6 @@ def call( Returns: Examples: - ```python >>> from transformers import ViTFeatureExtractor, TFViTForImageClassification >>> import tensorflow as tf @@ -787,6 +787,7 @@ def call( >>> # model predicts one of the 1000 ImageNet classes >>> predicted_class_idx = tf.math.argmax(logits, axis=-1)[0] >>> print("Predicted class:", model.config.id2label[int(predicted_class_idx)]) + Predicted class: Egyptian cat ```""" outputs = self.vit( diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 5e2435ef4618..3f9a20c17571 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -45,6 +45,7 @@ src/transformers/models/van/modeling_van.py src/transformers/models/vilt/modeling_vilt.py src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py src/transformers/models/vit/modeling_vit.py +src/transformers/models/vit/modeling_tf_vit.py src/transformers/models/vit_mae/modeling_vit_mae.py src/transformers/models/wav2vec2/modeling_wav2vec2.py src/transformers/models/wav2vec2/tokenization_wav2vec2.py From f9d231f15dbb3b04da8c24903c38736ed09da098 Mon Sep 17 00:00:00 2001 From: Johannes Kolbe Date: Mon, 28 Mar 2022 22:34:13 +0200 Subject: [PATCH 03/54] add empty lines back in --- src/transformers/models/vit/modeling_tf_vit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py index 36d3bbc69a07..1831d77754eb 100644 --- a/src/transformers/models/vit/modeling_tf_vit.py +++ b/src/transformers/models/vit/modeling_tf_vit.py @@ -662,6 +662,7 @@ def call( Returns: Examples: + ```python >>> from transformers import ViTFeatureExtractor, TFViTModel >>> from PIL import Image @@ -769,6 +770,7 @@ def call( Returns: Examples: + ```python >>> from transformers import ViTFeatureExtractor, TFViTForImageClassification >>> import tensorflow as tf From 85295621f129fc3ed5ea12628d10aadf7db9e85b Mon Sep 17 00:00:00 2001 From: Suraj Patil Date: Tue, 29 Mar 2022 11:32:13 +0200 Subject: [PATCH 04/54] Fix blenderbot conversion script (#16472) --- ...ert_blenderbot_original_pytorch_checkpoint_to_pytorch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/blenderbot/convert_blenderbot_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/blenderbot/convert_blenderbot_original_pytorch_checkpoint_to_pytorch.py index d31cf67c1e3f..c5919b94d42f 100644 --- a/src/transformers/models/blenderbot/convert_blenderbot_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/blenderbot/convert_blenderbot_original_pytorch_checkpoint_to_pytorch.py @@ -18,7 +18,7 @@ import torch -from transformers import BartConfig, BartForConditionalGeneration +from transformers import BlenderbotConfig, BlenderbotForConditionalGeneration from transformers.utils import logging @@ -81,8 +81,8 @@ def convert_parlai_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_ """ model = torch.load(checkpoint_path, map_location="cpu") sd = model["model"] - cfg = BartConfig.from_json_file(config_json_path) - m = BartForConditionalGeneration(cfg) + cfg = BlenderbotConfig.from_json_file(config_json_path) + m = BlenderbotForConditionalGeneration(cfg) valid_keys = m.model.state_dict().keys() failures = [] mapping = {} From ed31ab3f103e7b3b0b08658baa9b36174517fdd2 Mon Sep 17 00:00:00 2001 From: Arnaud Stiegler Date: Tue, 29 Mar 2022 10:19:06 -0400 Subject: [PATCH 05/54] Adding DocTest to TrOCR (#16398) * docstring still WIP | adding to documentation_tests * clean version | passes tests * adding to documentation_test * adding forward for training pass * make fixup applied * address comments * fix doctest * apply make fixup * remove additional blank * fix file to have correct split for prepare_for_doc_test * Update src/transformers/models/trocr/modeling_trocr.py Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> * address comments * changing text | adding loss check | make fixup * make fixup * Update src/transformers/models/trocr/modeling_trocr.py Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> * Update src/transformers/models/trocr/modeling_trocr.py Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> * Update src/transformers/models/trocr/modeling_trocr.py Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> * make fixup Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> --- .../models/trocr/modeling_trocr.py | 48 ++++++++++++++++--- utils/documentation_tests.txt | 1 + 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/trocr/modeling_trocr.py b/src/transformers/models/trocr/modeling_trocr.py index 8a26739d877f..75e015f98848 100644 --- a/src/transformers/models/trocr/modeling_trocr.py +++ b/src/transformers/models/trocr/modeling_trocr.py @@ -597,8 +597,8 @@ def forward( If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` - of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of + shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. @@ -891,13 +891,49 @@ def forward( Example: ```python - >>> from transformers import VisionEncoderDecoderModel, TrOCRForCausalLM, ViTModel, TrOCRConfig, ViTConfig - + >>> from transformers import ( + ... TrOCRConfig, + ... TrOCRProcessor, + ... TrOCRForCausalLM, + ... ViTConfig, + ... ViTModel, + ... VisionEncoderDecoderModel, + ... ) + >>> import requests + >>> from PIL import Image + + >>> # TrOCR is a decoder model and should be used within a VisionEncoderDecoderModel + >>> # init vision2text model with random weights >>> encoder = ViTModel(ViTConfig()) >>> decoder = TrOCRForCausalLM(TrOCRConfig()) - # init vision2text model - >>> model = VisionEncoderDecoderModel(encoder=encoder, decoder=decoder) + + >>> # If you want to start from the pretrained model, load the checkpoint with `VisionEncoderDecoderModel` + >>> processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") + >>> model = VisionEncoderDecoderModel.from_pretrained("microsoft/trocr-base-handwritten") + + >>> # load image from the IAM dataset + >>> url = "https://fki.tic.heia-fr.ch/static/img/a01-122-02.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB") + >>> pixel_values = processor(image, return_tensors="pt").pixel_values + >>> text = "industry, ' Mr. Brown commented icily. ' Let us have a" + + >>> # training + >>> model.config.decoder_start_token_id = processor.tokenizer.cls_token_id + >>> model.config.pad_token_id = processor.tokenizer.pad_token_id + >>> model.config.vocab_size = model.config.decoder.vocab_size + + >>> labels = processor.tokenizer(text, return_tensors="pt").input_ids + >>> outputs = model(pixel_values, labels=labels) + >>> loss = outputs.loss + >>> round(loss.item(), 2) + 5.30 + + >>> # inference + >>> generated_ids = model.generate(pixel_values) + >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + >>> generated_text + 'industry, " Mr. Brown commented icily. " Let us have a' ```""" output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 5e2435ef4618..372e63ad232b 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -39,6 +39,7 @@ src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.p src/transformers/models/speech_to_text/modeling_speech_to_text.py src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py src/transformers/models/swin/modeling_swin.py +src/transformers/models/trocr/modeling_trocr.py src/transformers/models/unispeech/modeling_unispeech.py src/transformers/models/unispeech_sat/modeling_unispeech_sat.py src/transformers/models/van/modeling_van.py From 5216607f8a67952395aca7b81b2dd9c7c60d2f4c Mon Sep 17 00:00:00 2001 From: Eldar Kurtic Date: Tue, 29 Mar 2022 16:38:14 +0200 Subject: [PATCH 06/54] [MNLI example] Prevent overwriting matched with mismatched metrics (#16475) * Prevent overwriting matched with mismatched metrics * Fix style --- examples/pytorch/text-classification/run_glue.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py index da0bb551de15..92c4d2b37936 100755 --- a/examples/pytorch/text-classification/run_glue.py +++ b/examples/pytorch/text-classification/run_glue.py @@ -507,6 +507,7 @@ def compute_metrics(p: EvalPrediction): if data_args.task_name == "mnli": tasks.append("mnli-mm") eval_datasets.append(raw_datasets["validation_mismatched"]) + combined = {} for eval_dataset, task in zip(eval_datasets, tasks): metrics = trainer.evaluate(eval_dataset=eval_dataset) @@ -516,8 +517,13 @@ def compute_metrics(p: EvalPrediction): ) metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) + if task == "mnli-mm": + metrics = {k + "_mm": v for k, v in metrics.items()} + if "mnli" in task: + combined.update(metrics) + trainer.log_metrics("eval", metrics) - trainer.save_metrics("eval", metrics) + trainer.save_metrics("eval", combined if "mnli" in task else metrics) if training_args.do_predict: logger.info("*** Predict ***") From 45abb37ac97b757cb25cdc83dadf91c1d243683c Mon Sep 17 00:00:00 2001 From: Steven Liu <59462357+stevhliu@users.noreply.github.com> Date: Tue, 29 Mar 2022 08:34:30 -0700 Subject: [PATCH 07/54] Remove duplicate mLuke (#16460) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove duplicate mLuke * 🖍 apply feedback --- docs/source/_toctree.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 0be8d4b9eca5..69717477e1f2 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -255,11 +255,9 @@ - local: model_doc/megatron_gpt2 title: MegatronGPT2 - local: model_doc/mluke - title: MLUKE + title: mLUKE - local: model_doc/mobilebert title: MobileBERT - - local: model_doc/mluke - title: mLUKE - local: model_doc/mpnet title: MPNet - local: model_doc/mt5 From aebca696afb511013d71fb6da5c162d945fb31d4 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 29 Mar 2022 17:51:48 +0200 Subject: [PATCH 08/54] Fix missing output_attentions in PT/Flax equivalence test (#16271) * fix - set output_attentions to True * Update tests/test_modeling_flax_common.py * update for has_attentions * overwrite check_outputs in FlaxBigBirdModelTest Co-authored-by: ydshieh Co-authored-by: Suraj Patil --- tests/big_bird/test_modeling_flax_big_bird.py | 9 +++++++++ tests/test_modeling_flax_common.py | 9 ++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/big_bird/test_modeling_flax_big_bird.py b/tests/big_bird/test_modeling_flax_big_bird.py index 834b71b30b24..5946129316b4 100644 --- a/tests/big_bird/test_modeling_flax_big_bird.py +++ b/tests/big_bird/test_modeling_flax_big_bird.py @@ -190,3 +190,12 @@ def model_jitted(input_ids, attention_mask=None, **kwargs): for jitted_output, output in zip(jitted_outputs, outputs): self.assertEqual(jitted_output.shape, output.shape) + + # overwrite from common in order to skip the check on `attentions` + def check_outputs(self, fx_outputs, pt_outputs, model_class, names): + # `bigbird_block_sparse_attention` in `FlaxBigBird` returns `attention_probs = None`, while in PyTorch version, + # an effort was done to return `attention_probs` (yet to be verified). + if type(names) == str and names.startswith("attentions"): + return + else: + super().check_outputs(fx_outputs, pt_outputs, model_class, names) diff --git a/tests/test_modeling_flax_common.py b/tests/test_modeling_flax_common.py index 0de5005fe6fa..e37352b976d1 100644 --- a/tests/test_modeling_flax_common.py +++ b/tests/test_modeling_flax_common.py @@ -120,6 +120,7 @@ class FlaxModelTesterMixin: test_mismatched_shapes = True is_encoder_decoder = False test_head_masking = False + has_attentions = True def _prepare_for_class(self, inputs_dict, model_class): inputs_dict = copy.deepcopy(inputs_dict) @@ -168,6 +169,7 @@ def recursive_check(tuple_object, dict_object): dict_inputs = self._prepare_for_class(inputs_dict, model_class) check_equivalence(model, tuple_inputs, dict_inputs, {"output_hidden_states": True}) + # (Copied from tests.test_modeling_common.ModelTesterMixin.check_outputs) def check_outputs(self, fx_outputs, pt_outputs, model_class, names): """ Args: @@ -204,8 +206,7 @@ def check_outputs(self, fx_outputs, pt_outputs, model_class, names): pt_outputs[pt_nans] = 0 fx_outputs[pt_nans] = 0 - max_diff = np.amax(np.abs(fx_outputs - pt_outputs)) - self.assertLessEqual(max_diff, 1e-5) + self.assert_almost_equals(fx_outputs, pt_outputs, 1e-5) else: raise ValueError( f"`fx_outputs` should be a `tuple` or an instance of `jnp.ndarray`. Got {type(fx_outputs)} instead." @@ -222,6 +223,7 @@ def test_equivalence_pt_to_flax(self): # Output all for aggressive testing config.output_hidden_states = True + config.output_attentions = self.has_attentions # prepare inputs prepared_inputs_dict = self._prepare_for_class(inputs_dict, model_class) @@ -274,7 +276,7 @@ def test_equivalence_flax_to_pt(self): # Output all for aggressive testing config.output_hidden_states = True - # Pure convolutional models have no attention + config.output_attentions = self.has_attentions # prepare inputs prepared_inputs_dict = self._prepare_for_class(inputs_dict, model_class) @@ -314,6 +316,7 @@ def test_equivalence_flax_to_pt(self): # send pytorch model to the correct device pt_model_loaded.to(torch_device) + pt_model_loaded.eval() with torch.no_grad(): pt_outputs_loaded = pt_model_loaded(**pt_inputs) From 86cff21cf60b86859934f6e6d867c95cceb8d6ac Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 29 Mar 2022 18:04:20 +0200 Subject: [PATCH 09/54] Fix some TF GPT-J CI testings (#16454) * Fix for test_mixed_precision * Fix test_saved_model_creation by using shape_list instead of shape * skit test_model_from_pretrained on GPU for now to avoid GPU OOM * skip test_gptj_sample_max_time for now Co-authored-by: ydshieh --- src/transformers/models/gptj/modeling_tf_gptj.py | 16 ++++++++-------- tests/gptj/test_modeling_tf_gptj.py | 5 +++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/gptj/modeling_tf_gptj.py b/src/transformers/models/gptj/modeling_tf_gptj.py index 6c24d747692f..4bb9ec4a81ed 100644 --- a/src/transformers/models/gptj/modeling_tf_gptj.py +++ b/src/transformers/models/gptj/modeling_tf_gptj.py @@ -59,13 +59,13 @@ def fixed_pos_embedding(x: tf.Tensor, seq_dim: int = 1, seq_len: Optional[int] = None) -> Tuple[tf.Tensor, tf.Tensor]: - dim = x.shape[-1] + dim = shape_list(x)[-1] if seq_len is None: - seq_len = x.shape[seq_dim] + seq_len = shape_list(x)[seq_dim] inv_freq = tf.cast(1.0 / (10000 ** (tf.range(0, dim, 2) / dim)), tf.float32) seq_len_range = tf.cast(tf.range(seq_len), tf.float32) sinusoid_inp = tf.cast(tf.einsum("i , j -> i j", seq_len_range, inv_freq), tf.float32) - return tf.sin(sinusoid_inp), tf.cos(sinusoid_inp) + return tf.cast(tf.sin(sinusoid_inp), dtype=x.dtype), tf.cast(tf.cos(sinusoid_inp), dtype=x.dtype) def rotate_every_two(x: tf.Tensor) -> tf.Tensor: @@ -77,8 +77,8 @@ def rotate_every_two(x: tf.Tensor) -> tf.Tensor: def apply_rotary_pos_emb(x: tf.Tensor, sincos: tf.Tensor, offset: int = 0) -> tf.Tensor: sin_pos, cos_pos = sincos - sin_pos = tf.repeat(sin_pos[None, offset : x.shape[1] + offset, None, :], 2, 3) - cos_pos = tf.repeat(cos_pos[None, offset : x.shape[1] + offset, None, :], 2, 3) + sin_pos = tf.repeat(sin_pos[None, offset : shape_list(x)[1] + offset, None, :], 2, 3) + cos_pos = tf.repeat(cos_pos[None, offset : shape_list(x)[1] + offset, None, :], 2, 3) return (x * cos_pos) + (rotate_every_two(x) * sin_pos) @@ -173,7 +173,7 @@ def _attn( head_mask: Optional[tf.Tensor] = None, ) -> Tuple[tf.Tensor, tf.Tensor]: # compute causal mask from causal mask buffer - query_length, key_length = query.shape[-2], key.shape[-2] + query_length, key_length = shape_list(query)[-2], shape_list(key)[-2] causal_mask = self.get_causal_mask(key_length, query_length) # Keep the attention weights computation in fp32 to avoid overflow issues @@ -218,11 +218,11 @@ def call( key = self._split_heads(key, True) value = self._split_heads(value, False) - seq_len = key.shape[1] + seq_len = shape_list(key)[1] offset = 0 if layer_past is not None: - offset = layer_past[0].shape[-2] + offset = shape_list(layer_past[0])[-2] seq_len += offset if self.rotary_dim is not None: diff --git a/tests/gptj/test_modeling_tf_gptj.py b/tests/gptj/test_modeling_tf_gptj.py index 50bcf1cc8a0f..32ce3f8564b0 100644 --- a/tests/gptj/test_modeling_tf_gptj.py +++ b/tests/gptj/test_modeling_tf_gptj.py @@ -345,6 +345,10 @@ def test_model_common_attributes(self): assert name is None @slow + @unittest.skipIf( + not is_tf_available() or len(tf.config.list_physical_devices("GPU")) > 0, + "skip testing on GPU for now to avoid GPU OOM.", + ) def test_model_from_pretrained(self): model = TFGPTJModel.from_pretrained("EleutherAI/gpt-j-6B", from_pt=True) self.assertIsNotNone(model) @@ -395,6 +399,7 @@ def test_gptj_sample(self): ) # token_type_ids should change output @slow + @unittest.skip(reason="TF generate currently has no time-based stopping criteria") def test_gptj_sample_max_time(self): tokenizer = AutoTokenizer.from_pretrained("anton-l/gpt-j-tiny-random") model = TFGPTJForCausalLM.from_pretrained("anton-l/gpt-j-tiny-random", from_pt=True) From b62ac4d240fb4d6d1b4040faf0ec168577b9f093 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 29 Mar 2022 12:21:19 -0400 Subject: [PATCH 10/54] Fix example test and test_fetcher for examples (#16478) --- examples/pytorch/text-classification/run_glue.py | 4 ++-- utils/tests_fetcher.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py index 92c4d2b37936..5f7ba4ca48c3 100755 --- a/examples/pytorch/text-classification/run_glue.py +++ b/examples/pytorch/text-classification/run_glue.py @@ -519,11 +519,11 @@ def compute_metrics(p: EvalPrediction): if task == "mnli-mm": metrics = {k + "_mm": v for k, v in metrics.items()} - if "mnli" in task: + if task is not None and "mnli" in task: combined.update(metrics) trainer.log_metrics("eval", metrics) - trainer.save_metrics("eval", combined if "mnli" in task else metrics) + trainer.save_metrics("eval", combined if task is not None and "mnli" in task else metrics) if training_args.do_predict: logger.info("*** Predict ***") diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index dae4fa421845..5ad7b4b1f788 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -465,9 +465,9 @@ def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None): test_files_to_run.append(f) # Example files are tested separately elif f.startswith("examples/pytorch"): - test_files_to_run.append("examples/pytorch/test_examples.py") + test_files_to_run.append("examples/pytorch/test_pytorch_examples.py") elif f.startswith("examples/flax"): - test_files_to_run.append("examples/flax/test_examples.py") + test_files_to_run.append("examples/flax/test_flax_examples.py") else: new_tests = module_to_test_file(f) if new_tests is not None: From 3015d12bfb8b5d65affe05476ee9fe636c7bba0e Mon Sep 17 00:00:00 2001 From: "Wesley A. Cheng" <15952538+wesleyacheng@users.noreply.github.com> Date: Tue, 29 Mar 2022 09:55:40 -0700 Subject: [PATCH 11/54] fix wrong variable name (#16467) --- docs/source/preprocessing.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/preprocessing.mdx b/docs/source/preprocessing.mdx index d6ffb0cf4a41..390acd72731d 100644 --- a/docs/source/preprocessing.mdx +++ b/docs/source/preprocessing.mdx @@ -175,7 +175,7 @@ Set the `return_tensors` parameter to either `pt` for PyTorch, or `tf` for Tenso ... "Don't think he knows about second breakfast, Pip.", ... "What about elevensies?", ... ] ->>> encoded_input = tokenizer(batch, padding=True, truncation=True, return_tensors="tf") +>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf") >>> print(encoded_input) {'input_ids': Date: Tue, 29 Mar 2022 18:57:16 +0200 Subject: [PATCH 12/54] Add TF vision model code samples (#16477) * add code samples Co-authored-by: ydshieh --- src/transformers/utils/doc.py | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/transformers/utils/doc.py b/src/transformers/utils/doc.py index 394d2aaa2fed..f395f8d4fb80 100644 --- a/src/transformers/utils/doc.py +++ b/src/transformers/utils/doc.py @@ -794,6 +794,52 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): ``` """ +TF_VISION_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import {processor_class}, {model_class} + >>> from datasets import load_dataset + + >>> dataset = load_dataset("huggingface/cats-image") + >>> image = dataset["test"]["image"][0] + + >>> feature_extractor = {processor_class}.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = feature_extractor(image, return_tensors="tf") + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + {expected_output} + ``` +""" + +TF_VISION_SEQ_CLASS_SAMPLE = r""" + Example: + + ```python + >>> from transformers import {processor_class}, {model_class} + >>> import tensorflow as tf + >>> from datasets import load_dataset + + >>> dataset = load_dataset("huggingface/cats-image") + >>> image = dataset["test"]["image"][0] + + >>> feature_extractor = {processor_class}.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> inputs = feature_extractor(image, return_tensors="tf") + >>> logits = model(**inputs).logits + + >>> # model predicts one of the 1000 ImageNet classes + >>> predicted_label = int(tf.math.argmax(logits, axis=-1)) + >>> print(model.config.id2label[predicted_label]) + {expected_output} + ``` +""" + TF_SAMPLE_DOCSTRINGS = { "SequenceClassification": TF_SEQUENCE_CLASSIFICATION_SAMPLE, "QuestionAnswering": TF_QUESTION_ANSWERING_SAMPLE, @@ -802,6 +848,8 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): "MaskedLM": TF_MASKED_LM_SAMPLE, "LMHead": TF_CAUSAL_LM_SAMPLE, "BaseModel": TF_BASE_MODEL_SAMPLE, + "VisionBaseModel": TF_VISION_BASE_MODEL_SAMPLE, + "ImageClassification": TF_VISION_SEQ_CLASS_SAMPLE, } From 875e07a9e3e8aac3f01b607d6c4f58678db8188a Mon Sep 17 00:00:00 2001 From: "Wesley A. Cheng" <15952538+wesleyacheng@users.noreply.github.com> Date: Tue, 29 Mar 2022 09:57:43 -0700 Subject: [PATCH 13/54] [doc] Fix missing trainer import (#16469) --- docs/source/training.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/training.mdx b/docs/source/training.mdx index e6c4b146fac6..0d3648a92551 100644 --- a/docs/source/training.mdx +++ b/docs/source/training.mdx @@ -121,7 +121,7 @@ Call `compute` on `metric` to calculate the accuracy of your predictions. Before If you'd like to monitor your evaluation metrics during fine-tuning, specify the `evaluation_strategy` parameter in your training arguments to report the evaluation metric at the end of each epoch: ```py ->>> from transformers import TrainingArguments +>>> from transformers import TrainingArguments, Trainer >>> training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch") ``` From 0540d1b6c0bc1eae80144c5d228e89e87589ea99 Mon Sep 17 00:00:00 2001 From: Dan Tegzes <48134725+Tegzes@users.noreply.github.com> Date: Tue, 29 Mar 2022 19:02:46 +0200 Subject: [PATCH 14/54] Add type hints for UniSpeech (#16399) * Add type hints for UniSpeech * Added type hints for UniSpeechSat * Added type hints for Wave2Vec2 (PT) * Added type hints for models dependent of wave2vec --- .../data2vec/modeling_data2vec_audio.py | 68 +++++++-------- .../models/hubert/modeling_hubert.py | 42 +++++----- src/transformers/models/sew/modeling_sew.py | 42 +++++----- .../models/sew_d/modeling_sew_d.py | 42 +++++----- .../models/unispeech/modeling_unispeech.py | 54 ++++++------ .../unispeech_sat/modeling_unispeech_sat.py | 80 +++++++++--------- .../models/wav2vec2/modeling_wav2vec2.py | 84 +++++++++---------- .../models/wavlm/modeling_wavlm.py | 68 +++++++-------- 8 files changed, 240 insertions(+), 240 deletions(-) diff --git a/src/transformers/models/data2vec/modeling_data2vec_audio.py b/src/transformers/models/data2vec/modeling_data2vec_audio.py index 1b531c048961..3f255248c1ba 100755 --- a/src/transformers/models/data2vec/modeling_data2vec_audio.py +++ b/src/transformers/models/data2vec/modeling_data2vec_audio.py @@ -977,13 +977,13 @@ def _mask_hidden_states( ) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, Data2VecAudioBaseModelOutput]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1085,13 +1085,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1221,13 +1221,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1342,12 +1342,12 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, TokenClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1519,13 +1519,13 @@ def _conv_out_length(input_length, kernel_size, stride): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, XVectorOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., diff --git a/src/transformers/models/hubert/modeling_hubert.py b/src/transformers/models/hubert/modeling_hubert.py index 00ac994dd632..5af0197fb95c 100755 --- a/src/transformers/models/hubert/modeling_hubert.py +++ b/src/transformers/models/hubert/modeling_hubert.py @@ -1008,13 +1008,13 @@ def _mask_hidden_states( @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: """ Returns: @@ -1135,13 +1135,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1271,13 +1271,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., diff --git a/src/transformers/models/sew/modeling_sew.py b/src/transformers/models/sew/modeling_sew.py index e3561f780b5b..ac2a6293cb95 100644 --- a/src/transformers/models/sew/modeling_sew.py +++ b/src/transformers/models/sew/modeling_sew.py @@ -914,13 +914,13 @@ def _mask_hidden_states( ) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1018,13 +1018,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1154,13 +1154,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., diff --git a/src/transformers/models/sew_d/modeling_sew_d.py b/src/transformers/models/sew_d/modeling_sew_d.py index 4470a10ebcd1..a297e4c7b25b 100644 --- a/src/transformers/models/sew_d/modeling_sew_d.py +++ b/src/transformers/models/sew_d/modeling_sew_d.py @@ -1427,13 +1427,13 @@ def _mask_hidden_states( ) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1531,13 +1531,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1667,13 +1667,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py index e7206b587524..94f8f0e3d2e6 100755 --- a/src/transformers/models/unispeech/modeling_unispeech.py +++ b/src/transformers/models/unispeech/modeling_unispeech.py @@ -1160,13 +1160,13 @@ def _mask_hidden_states( ) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, UniSpeechBaseModelOutput]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1274,12 +1274,12 @@ def compute_contrastive_logits( @replace_return_docstrings(output_type=UniSpeechForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, UniSpeechForPreTrainingOutput]: r""" mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict @@ -1412,13 +1412,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1548,13 +1548,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., diff --git a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py index ae5f57d8a1e5..ea010b264794 100755 --- a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py +++ b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py @@ -1199,13 +1199,13 @@ def _mask_hidden_states( ) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, UniSpeechSatBaseModelOutput]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1318,12 +1318,12 @@ def compute_contrastive_logits( @replace_return_docstrings(output_type=UniSpeechSatForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, UniSpeechSatForPreTrainingOutput]: r""" Returns: @@ -1440,13 +1440,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1576,13 +1576,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1697,12 +1697,12 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, TokenClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1874,13 +1874,13 @@ def _conv_out_length(input_length, kernel_size, stride): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, XVectorOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., diff --git a/src/transformers/models/wav2vec2/modeling_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_wav2vec2.py index 08a86d540311..f58ec9a3363e 100755 --- a/src/transformers/models/wav2vec2/modeling_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_wav2vec2.py @@ -1331,13 +1331,13 @@ def _mask_hidden_states( ) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, Wav2Vec2BaseModelOutput]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1448,14 +1448,14 @@ def compute_contrastive_logits( @replace_return_docstrings(output_type=Wav2Vec2ForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - sampled_negative_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.BoolTensor] = None, + sampled_negative_indices: Optional[torch.BoolTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, Wav2Vec2ForPreTrainingOutput]: r""" mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict @@ -1696,13 +1696,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1831,13 +1831,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1951,12 +1951,12 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, TokenClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -2125,13 +2125,13 @@ def _conv_out_length(input_length, kernel_size, stride): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, XVectorOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., diff --git a/src/transformers/models/wavlm/modeling_wavlm.py b/src/transformers/models/wavlm/modeling_wavlm.py index e6b47056ce2e..c2eb193160ad 100755 --- a/src/transformers/models/wavlm/modeling_wavlm.py +++ b/src/transformers/models/wavlm/modeling_wavlm.py @@ -1282,13 +1282,13 @@ def _mask_hidden_states( ) def forward( self, - input_values, - attention_mask=None, - mask_time_indices=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + mask_time_indices: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, WavLMBaseModelOutput]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1390,13 +1390,13 @@ def freeze_feature_encoder(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, CausalLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to @@ -1526,13 +1526,13 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1647,12 +1647,12 @@ def freeze_base_model(self): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, TokenClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1824,13 +1824,13 @@ def _conv_out_length(input_length, kernel_size, stride): ) def forward( self, - input_values, - attention_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_values: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.Tensor] = None, + ) -> Union[Tuple, XVectorOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., From 7a9ef8181c9bb92385eccbb2d35e864fa80fadf2 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Tue, 29 Mar 2022 18:17:47 +0100 Subject: [PATCH 15/54] TF: properly handle kwargs in encoder_decoder architectures (#16465) * properly handle kwargs in encoder_decoder architectures * make fixup --- .../models/encoder_decoder/modeling_tf_encoder_decoder.py | 6 ++---- .../modeling_tf_vision_encoder_decoder.py | 6 ++---- tests/encoder_decoder/test_modeling_tf_encoder_decoder.py | 8 ++++++++ .../test_modeling_tf_vision_encoder_decoder.py | 8 ++++++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py index c2be91c7a000..1c59493e1bf7 100644 --- a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py @@ -569,13 +569,12 @@ def call( "output_hidden_states": output_hidden_states, "return_dict": return_dict, "training": training, - "kwargs_call": kwargs_encoder, + "kwargs_call": {}, } # Add arguments to encoder from `kwargs_encoder` for k, v in kwargs_encoder.items(): encoder_processing_inputs[k] = v - kwargs_encoder = {} encoder_inputs = input_processing(**encoder_processing_inputs) @@ -622,13 +621,12 @@ def call( "past_key_values": past_key_values, "return_dict": return_dict, "training": training, - "kwargs_call": kwargs_decoder, + "kwargs_call": {}, } # Add arguments to decoder from `kwargs_decoder` for k, v in kwargs_decoder.items(): decoder_processing_inputs[k] = v - kwargs_decoder = {} decoder_inputs = input_processing(**decoder_processing_inputs) decoder_outputs = self.decoder(**decoder_inputs) diff --git a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py index 965fc51d783b..eeaca58c5a01 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py @@ -593,12 +593,11 @@ def call( "output_hidden_states": output_hidden_states, "return_dict": return_dict, "training": training, - "kwargs_call": kwargs_encoder, + "kwargs_call": {}, } # Add arguments to encoder from `kwargs_encoder` encoder_processing_inputs.update(kwargs_encoder) - kwargs_encoder = {} encoder_inputs = input_processing(**encoder_processing_inputs) @@ -654,12 +653,11 @@ def call( "past_key_values": past_key_values, "return_dict": return_dict, "training": training, - "kwargs_call": kwargs_decoder, + "kwargs_call": {}, } # Add arguments to decoder from `kwargs_decoder` decoder_processing_inputs.update(kwargs_decoder) - kwargs_decoder = {} decoder_inputs = input_processing(**decoder_processing_inputs) decoder_outputs = self.decoder(**decoder_inputs) diff --git a/tests/encoder_decoder/test_modeling_tf_encoder_decoder.py b/tests/encoder_decoder/test_modeling_tf_encoder_decoder.py index edcc881f564a..de903c40c26e 100644 --- a/tests/encoder_decoder/test_modeling_tf_encoder_decoder.py +++ b/tests/encoder_decoder/test_modeling_tf_encoder_decoder.py @@ -91,6 +91,7 @@ def check_encoder_decoder_model_from_pretrained_configs( decoder_input_ids=decoder_input_ids, attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) self.assertEqual( @@ -122,6 +123,7 @@ def check_encoder_decoder_model( decoder_input_ids=decoder_input_ids, attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) self.assertEqual( outputs_encoder_decoder["logits"].shape, (decoder_input_ids.shape + (decoder_config.vocab_size,)) @@ -137,6 +139,7 @@ def check_encoder_decoder_model( decoder_input_ids=decoder_input_ids, attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) self.assertEqual( @@ -167,6 +170,7 @@ def check_encoder_decoder_model_from_pretrained( attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, return_dict=True, + kwargs=kwargs, ) self.assertEqual( @@ -195,6 +199,7 @@ def check_save_and_load( decoder_input_ids=decoder_input_ids, attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) out_2 = np.array(outputs[0]) out_2[np.isnan(out_2)] = 0 @@ -208,6 +213,7 @@ def check_save_and_load( decoder_input_ids=decoder_input_ids, attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) out_1 = np.array(after_outputs[0]) out_1[np.isnan(out_1)] = 0 @@ -235,6 +241,7 @@ def check_encoder_decoder_model_labels( attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, labels=labels, + kwargs=kwargs, ) # Make sure `loss` exist @@ -269,6 +276,7 @@ def check_encoder_decoder_model_output_attentions( attention_mask=attention_mask, decoder_attention_mask=decoder_attention_mask, output_attentions=True, + kwargs=kwargs, ) encoder_attentions = outputs_encoder_decoder["encoder_attentions"] diff --git a/tests/vision_encoder_decoder/test_modeling_tf_vision_encoder_decoder.py b/tests/vision_encoder_decoder/test_modeling_tf_vision_encoder_decoder.py index a0fcbfaea325..f3a062744f5c 100644 --- a/tests/vision_encoder_decoder/test_modeling_tf_vision_encoder_decoder.py +++ b/tests/vision_encoder_decoder/test_modeling_tf_vision_encoder_decoder.py @@ -96,6 +96,7 @@ def check_encoder_decoder_model_from_pretrained_configs( pixel_values=pixel_values, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) self.assertEqual( @@ -124,6 +125,7 @@ def check_encoder_decoder_model( pixel_values=pixel_values, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) self.assertEqual( outputs_encoder_decoder["logits"].shape, (decoder_input_ids.shape + (decoder_config.vocab_size,)) @@ -137,6 +139,7 @@ def check_encoder_decoder_model( encoder_outputs=encoder_outputs, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) self.assertEqual( @@ -164,6 +167,7 @@ def check_encoder_decoder_model_from_pretrained( decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, return_dict=True, + kwargs=kwargs, ) self.assertEqual( @@ -189,6 +193,7 @@ def check_save_and_load( pixel_values=pixel_values, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) out_2 = np.array(outputs[0]) out_2[np.isnan(out_2)] = 0 @@ -201,6 +206,7 @@ def check_save_and_load( pixel_values=pixel_values, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, + kwargs=kwargs, ) out_1 = np.array(after_outputs[0]) out_1[np.isnan(out_1)] = 0 @@ -226,6 +232,7 @@ def check_encoder_decoder_model_labels( decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, labels=labels, + kwargs=kwargs, ) # Make sure `loss` exist @@ -257,6 +264,7 @@ def check_encoder_decoder_model_output_attentions( decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, output_attentions=True, + kwargs=kwargs, ) encoder_attentions = outputs_encoder_decoder["encoder_attentions"] From 5b40a37bc4da9dc6cd33876ce9bb3f7f48450a03 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Tue, 29 Mar 2022 22:54:15 +0530 Subject: [PATCH 16/54] Add TF ViT MAE (#16255) * ported TFViTMAEIntermediate and TFViTMAEOutput. * added TFViTMAEModel and TFViTMAEDecoder. * feat: added a noise argument in the implementation for reproducibility. * feat: vit mae models with an additional noise argument for reproducibility. Co-authored-by: ariG23498 Co-authored-by: ydshieh --- docs/source/index.mdx | 2 +- docs/source/model_doc/vit_mae.mdx | 17 +- src/transformers/__init__.py | 8 + .../models/auto/modeling_tf_auto.py | 2 + src/transformers/models/vit_mae/__init__.py | 9 + .../models/vit_mae/modeling_tf_vit_mae.py | 1087 +++++++++++++++++ .../models/vit_mae/modeling_vit_mae.py | 16 +- src/transformers/utils/dummy_tf_objects.py | 21 + tests/vit_mae/test_modeling_tf_vit_mae.py | 718 +++++++++++ tests/vit_mae/test_modeling_vit_mae.py | 181 ++- 10 files changed, 2034 insertions(+), 27 deletions(-) create mode 100644 src/transformers/models/vit_mae/modeling_tf_vit_mae.py create mode 100644 tests/vit_mae/test_modeling_tf_vit_mae.py diff --git a/docs/source/index.mdx b/docs/source/index.mdx index af1608dd4ce2..281add6e5ef4 100644 --- a/docs/source/index.mdx +++ b/docs/source/index.mdx @@ -260,7 +260,7 @@ Flax), PyTorch, and/or TensorFlow. | VisionTextDualEncoder | ❌ | ❌ | ✅ | ❌ | ✅ | | VisualBert | ❌ | ❌ | ✅ | ❌ | ❌ | | ViT | ❌ | ❌ | ✅ | ✅ | ✅ | -| ViTMAE | ❌ | ❌ | ✅ | ❌ | ❌ | +| ViTMAE | ❌ | ❌ | ✅ | ✅ | ❌ | | Wav2Vec2 | ✅ | ❌ | ✅ | ✅ | ✅ | | WavLM | ❌ | ❌ | ✅ | ❌ | ❌ | | XGLM | ✅ | ✅ | ✅ | ❌ | ✅ | diff --git a/docs/source/model_doc/vit_mae.mdx b/docs/source/model_doc/vit_mae.mdx index c320c1d55d27..aeb19b96a154 100644 --- a/docs/source/model_doc/vit_mae.mdx +++ b/docs/source/model_doc/vit_mae.mdx @@ -41,13 +41,16 @@ fine-tuning, one can directly plug in the weights into a [`ViTForImageClassifica - Note that the encoder of MAE is only used to encode the visual patches. The encoded patches are then concatenated with mask tokens, which the decoder (which also consists of Transformer blocks) takes as input. Each mask token is a shared, learned vector that indicates the presence of a missing patch to be predicted. Fixed sin/cos position embeddings are added both to the input of the encoder and the decoder. +- For a visual understanding of how MAEs work you can check out this [post](https://keras.io/examples/vision/masked_image_modeling/). drawing MAE architecture. Taken from the original paper. -This model was contributed by [nielsr](https://huggingface.co/nielsr). The original code can be found [here](https://github.com/facebookresearch/mae). +This model was contributed by [nielsr](https://huggingface.co/nielsr). TensorFlow version of the model was contributed by [sayakpaul](https://github.com/sayakpaul) and +[ariG23498](https://github.com/ariG23498) (equal contribution). The original code can be found [here](https://github.com/facebookresearch/mae). + ## ViTMAEConfig @@ -64,3 +67,15 @@ This model was contributed by [nielsr](https://huggingface.co/nielsr). The origi [[autodoc]] transformers.ViTMAEForPreTraining - forward + + +## TFViTMAEModel + +[[autodoc]] TFViTMAEModel + - call + + +## TFViTMAEForPreTraining + +[[autodoc]] transformers.TFViTMAEForPreTraining + - call diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 0760618c69ea..841b830c3b1d 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -2135,6 +2135,13 @@ "TFViTPreTrainedModel", ] ) + _import_structure["models.vit_mae"].extend( + [ + "TFViTMAEForPreTraining", + "TFViTMAEModel", + "TFViTMAEPreTrainedModel", + ] + ) _import_structure["models.wav2vec2"].extend( [ "TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -4170,6 +4177,7 @@ ) from .models.vision_encoder_decoder import TFVisionEncoderDecoderModel from .models.vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel + from .models.vit_mae import TFViTMAEForPreTraining, TFViTMAEModel, TFViTMAEPreTrainedModel from .models.wav2vec2 import ( TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST, TFWav2Vec2ForCTC, diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py index 8afa05ba5785..a5cbb4852536 100644 --- a/src/transformers/models/auto/modeling_tf_auto.py +++ b/src/transformers/models/auto/modeling_tf_auto.py @@ -70,6 +70,7 @@ ("blenderbot", "TFBlenderbotModel"), ("blenderbot-small", "TFBlenderbotSmallModel"), ("vit", "TFViTModel"), + ("vit_mae", "TFViTMAEModel"), ("wav2vec2", "TFWav2Vec2Model"), ("hubert", "TFHubertModel"), ] @@ -100,6 +101,7 @@ ("tapas", "TFTapasForMaskedLM"), ("funnel", "TFFunnelForPreTraining"), ("mpnet", "TFMPNetForMaskedLM"), + ("vit_mae", "TFViTMAEForPreTraining"), ] ) diff --git a/src/transformers/models/vit_mae/__init__.py b/src/transformers/models/vit_mae/__init__.py index 02ad4db31945..cc3569b8b7f6 100644 --- a/src/transformers/models/vit_mae/__init__.py +++ b/src/transformers/models/vit_mae/__init__.py @@ -33,6 +33,12 @@ "ViTMAEPreTrainedModel", ] +if is_tf_available(): + _import_structure["modeling_tf_vit_mae"] = [ + "TFViTMAEForPreTraining", + "TFViTMAEModel", + "TFViTMAEPreTrainedModel", + ] if TYPE_CHECKING: from .configuration_vit_mae import VIT_MAE_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTMAEConfig @@ -46,6 +52,9 @@ ViTMAEPreTrainedModel, ) + if is_tf_available(): + from .modeling_tf_vit_mae import TFViTMAEForPreTraining, TFViTMAEModel, TFViTMAEPreTrainedModel + else: import sys diff --git a/src/transformers/models/vit_mae/modeling_tf_vit_mae.py b/src/transformers/models/vit_mae/modeling_tf_vit_mae.py new file mode 100644 index 000000000000..40f100b64ff1 --- /dev/null +++ b/src/transformers/models/vit_mae/modeling_tf_vit_mae.py @@ -0,0 +1,1087 @@ +# coding=utf-8 +# Copyright 2022 Facebook AI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 ViT MAE (masked autoencoder) model.""" + +import collections.abc +import math +from copy import deepcopy +from dataclasses import dataclass +from typing import Dict, Optional, Tuple, Union + +import numpy as np +import tensorflow as tf + +from ...activations_tf import get_tf_activation +from ...file_utils import ( + ModelOutput, + add_start_docstrings, + add_start_docstrings_to_model_forward, + replace_return_docstrings, +) +from ...modeling_tf_outputs import TFBaseModelOutput +from ...modeling_tf_utils import ( + TFModelInputType, + TFPreTrainedModel, + get_initializer, + keras_serializable, + unpack_inputs, +) +from ...tf_utils import shape_list +from ...utils import logging +from .configuration_vit_mae import ViTMAEConfig + + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "ViTMAEConfig" +_CHECKPOINT_FOR_DOC = "facebook/vit-mae-base" + + +@dataclass +class TFViTMAEModelOutput(ModelOutput): + """ + Class for TFViTMAEModel's outputs, with potential hidden states and attentions. + + Args: + last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + mask (`tf.Tensor` of shape `(batch_size, sequence_length)`): + Tensor indicating which patches are masked (1) and which are not (0). + ids_restore (`tf.Tensor` of shape `(batch_size, sequence_length)`): + Tensor containing the original index of the (shuffled) masked patches. + hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape + `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus + the initial embedding outputs. + attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in + the self-attention heads. + """ + + last_hidden_state: tf.Tensor = None + mask: tf.Tensor = None + ids_restore: tf.Tensor = None + hidden_states: Optional[Tuple[tf.Tensor]] = None + attentions: Optional[Tuple[tf.Tensor]] = None + + +@dataclass +class TFViTMAEDecoderOutput(ModelOutput): + """ + Class for TFViTMAEDecoder's outputs, with potential hidden states and attentions. + + Args: + logits (`tf.Tensor` of shape `(batch_size, patch_size ** 2 * num_channels)`): + Pixel reconstruction logits. + hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape + `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus + the initial embedding outputs. + attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in + the self-attention heads. + """ + + logits: tf.Tensor = None + hidden_states: Optional[Tuple[tf.Tensor]] = None + attentions: Optional[Tuple[tf.Tensor]] = None + + +@dataclass +class TFViTMAEForPreTrainingOutput(ModelOutput): + """ + Class for TFViTMAEForPreTraining's outputs, with potential hidden states and attentions. + + Args: + loss (`tf.Tensor` of shape `(1,)`): + Pixel reconstruction loss. + logits (`tf.Tensor` of shape `(batch_size, patch_size ** 2 * num_channels)`): + Pixel reconstruction logits. + mask (`tf.Tensor` of shape `(batch_size, sequence_length)`): + Tensor indicating which patches are masked (1) and which are not (0). + ids_restore (`tf.Tensor` of shape `(batch_size, sequence_length)`): + Tensor containing the original index of the (shuffled) masked patches. + hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape + `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus + the initial embedding outputs. + attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in + the self-attention heads. + """ + + loss: Optional[tf.Tensor] = None + logits: tf.Tensor = None + mask: tf.Tensor = None + ids_restore: tf.Tensor = None + hidden_states: Optional[Tuple[tf.Tensor]] = None + attentions: Optional[Tuple[tf.Tensor]] = None + + +# copied from transformers.models.vit.modeling_tf_vit.to_2tuple +def to_2tuple(x): + if isinstance(x, collections.abc.Iterable): + return x + return (x, x) + + +def get_2d_sincos_pos_embed(embed_dim, grid_size, add_cls_token=False): + """ + Create 2D sin/cos positional embeddings. + + Args: + embed_dim (`int`): + Embedding dimension. + grid_size (`int`): + The grid height and width. + add_cls_token (`bool`, *optional*, defaults to `False`): + Whether or not to add a classification (CLS) token. + + Returns: + (`tf.Tensor` of shape (grid_size*grid_size, embed_dim) or (1+grid_size*grid_size, embed_dim): the position + embeddings (with or without classification token) + """ + grid_h = tf.range(grid_size, dtype=tf.float32) + grid_w = tf.range(grid_size, dtype=tf.float32) + grid = tf.meshgrid(grid_w, grid_h) # here w goes first + grid = tf.stack(grid, axis=0) + + grid = tf.reshape(grid, [2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if add_cls_token: + pos_embed = tf.concat([tf.zeros((1, embed_dim)), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + if embed_dim % 2 != 0: + raise ValueError("embed_dim must be even") + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = tf.concat([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position pos: a list of positions to be encoded: size (M,) out: (M, D) + """ + if embed_dim % 2 != 0: + raise ValueError("embed_dim must be even") + + omega = tf.range(embed_dim // 2, dtype="float32") + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = tf.reshape(pos, [-1]) # (M,) + out = tf.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + # half of the positions get sinusoidal pattern and the rest gets + # cosine pattern and then they are concatenated + emb_sin = tf.sin(out) # (M, D/2) + emb_cos = tf.cos(out) # (M, D/2) + + emb = tf.concat([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +class TFViTMAEEmbeddings(tf.keras.layers.Layer): + """ + Construct the CLS token, position and patch embeddings. + + """ + + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.patch_embeddings = TFPatchEmbeddings(config, name="patch_embeddings") + self.num_patches = self.patch_embeddings.num_patches + + self.config = config + + def build(self, input_shape: tf.TensorShape): + self.cls_token = self.add_weight( + shape=(1, 1, self.config.hidden_size), + initializer=tf.random_normal_initializer(stddev=self.config.initializer_range), + trainable=True, + name="cls_token", + ) + self.position_embeddings = self.add_weight( + shape=(1, self.num_patches + 1, self.config.hidden_size), + initializer="zeros", + trainable=False, # fixed sin-cos embedding + name="position_embeddings", + ) + pos_embed = get_2d_sincos_pos_embed( + self.position_embeddings.shape[-1], + int(self.patch_embeddings.num_patches**0.5), + add_cls_token=True, + )[None, ...] + self.position_embeddings.assign(pos_embed) + + super().build(input_shape) + + def random_masking(self, sequence: tf.Tensor, noise: Optional[tf.Tensor] = None): + """ + Perform per-sample random masking by per-sample shuffling. Per-sample shuffling is done by argsort random + noise. + + Args: + sequence (`tf.Tensor` of shape `(batch_size, sequence_length, dim)`) + noise (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*) which is + mainly used for testing purposes to control randomness and maintain the reproducibility + """ + batch_size, seq_length, dim = shape_list(sequence) + len_keep = int(seq_length * (1 - self.config.mask_ratio)) + + if noise is None: + noise = tf.random.uniform(shape=(batch_size, seq_length), minval=0.0, maxval=1.0) # noise in [0, 1) + + # sort noise for each sample + ids_shuffle = tf.argsort(noise, axis=1) # ascend: small is keep, large is remove + ids_restore = tf.argsort(ids_shuffle, axis=1) + + # keep the first subset + ids_keep = ids_shuffle[:, :len_keep] + sequence_masked = tf.gather( + sequence, + axis=1, + batch_dims=1, + indices=ids_keep, + ) + + # generate the binary mask: 0 is keep, 1 is remove + # this hack is needed because TF's EagerTensors don't support + # assignment + mask_keep = tf.zeros((batch_size, len_keep)) + mask_remove = tf.ones((batch_size, seq_length - len_keep)) + mask = tf.concat([mask_keep, mask_remove], axis=-1) + + # unshuffle to get the binary mask + mask = tf.gather(mask, axis=1, batch_dims=1, indices=ids_restore) + + return sequence_masked, mask, ids_restore + + def call(self, pixel_values: tf.Tensor, noise: tf.Tensor = None) -> tf.Tensor: + embeddings = self.patch_embeddings(pixel_values) + + # add position embeddings w/o cls token + embeddings = embeddings + self.position_embeddings[:, 1:, :] + + # masking: length -> length * config.mask_ratio + embeddings, mask, ids_restore = self.random_masking(embeddings, noise) + + # append cls token + cls_token = self.cls_token + self.position_embeddings[:, :1, :] + cls_tokens = tf.tile(cls_token, (shape_list(embeddings)[0], 1, 1)) + embeddings = tf.concat([cls_tokens, embeddings], axis=1) + + return embeddings, mask, ids_restore + + +# Based on timm implementation, which can be found here: +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py +class TFPatchEmbeddings(tf.keras.layers.Layer): + """ + Image to Patch Embedding. + + """ + + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + image_size = to_2tuple(config.image_size) + patch_size = to_2tuple(config.patch_size) + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + self.image_size = image_size + self.patch_size = patch_size + self.num_patches = num_patches + self.num_channels = config.num_channels + self.embed_dim = config.hidden_size + self.config = config + + self.projection = tf.keras.layers.Conv2D( + filters=self.embed_dim, + kernel_size=self.patch_size, + strides=self.patch_size, + padding="valid", + data_format="channels_last", + kernel_initializer="glorot_uniform", # following torch.nn.Linear + bias_initializer="zeros", + name="projection", + ) + + def call(self, pixel_values: tf.Tensor, training: bool = False) -> tf.Tensor: + batch_size, num_channels, height, width = shape_list(pixel_values) + if getattr(height, "numpy", None) and getattr(width, "numpy", None): + if height != self.image_size[0] or width != self.image_size[1]: + raise ValueError( + f"Input image size ({height}*{width}) doesn't match model ({self.image_size[0]}*{self.image_size[1]})." + ) + + # When running on CPU, `tf.keras.layers.Conv2D` doesn't support `NCHW` format. + # So change the input format from `NCHW` to `NHWC`. + # shape = (batch_size, in_height, in_width, in_channels=num_channels) + pixel_values = tf.transpose(pixel_values, perm=(0, 2, 3, 1)) + + projection = self.projection(pixel_values) + + # Change the 2D spatial dimensions to a single temporal dimension. + # shape = (batch_size, num_patches, out_channels=embed_dim) + num_patches = (width // self.patch_size[1]) * (height // self.patch_size[0]) + x = tf.reshape(tensor=projection, shape=(batch_size, num_patches, -1)) + + return x + + +# Copied from transformers.models.vit.modeling_tf_vit.TFViTSelfAttention with ViT->ViTMAE +class TFViTMAESelfAttention(tf.keras.layers.Layer): + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + f"The hidden size ({config.hidden_size}) is not a multiple of the number " + f"of attention heads ({config.num_attention_heads})" + ) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + self.sqrt_att_head_size = math.sqrt(self.attention_head_size) + + self.query = tf.keras.layers.Dense( + units=self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="query" + ) + self.key = tf.keras.layers.Dense( + units=self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="key" + ) + self.value = tf.keras.layers.Dense( + units=self.all_head_size, kernel_initializer=get_initializer(config.initializer_range), name="value" + ) + self.dropout = tf.keras.layers.Dropout(rate=config.attention_probs_dropout_prob) + + def transpose_for_scores(self, tensor: tf.Tensor, batch_size: int) -> tf.Tensor: + # Reshape from [batch_size, seq_length, all_head_size] to [batch_size, seq_length, num_attention_heads, attention_head_size] + tensor = tf.reshape(tensor=tensor, shape=(batch_size, -1, self.num_attention_heads, self.attention_head_size)) + + # Transpose the tensor from [batch_size, seq_length, num_attention_heads, attention_head_size] to [batch_size, num_attention_heads, seq_length, attention_head_size] + return tf.transpose(tensor, perm=[0, 2, 1, 3]) + + def call( + self, + hidden_states: tf.Tensor, + head_mask: tf.Tensor, + output_attentions: bool, + training: bool = False, + ) -> Tuple[tf.Tensor]: + batch_size = shape_list(hidden_states)[0] + mixed_query_layer = self.query(inputs=hidden_states) + mixed_key_layer = self.key(inputs=hidden_states) + mixed_value_layer = self.value(inputs=hidden_states) + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + dk = tf.cast(self.sqrt_att_head_size, dtype=attention_scores.dtype) + attention_scores = tf.divide(attention_scores, dk) + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(logits=attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(inputs=attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = tf.multiply(attention_probs, head_mask) + + attention_output = tf.matmul(attention_probs, value_layer) + attention_output = tf.transpose(attention_output, perm=[0, 2, 1, 3]) + + # (batch_size, seq_len_q, all_head_size) + attention_output = tf.reshape(tensor=attention_output, shape=(batch_size, -1, self.all_head_size)) + outputs = (attention_output, attention_probs) if output_attentions else (attention_output,) + + return outputs + + +# Copied from transformers.models.vit.modeling_tf_vit.TFViTSelfOutput with ViT->ViTMAE +class TFViTMAESelfOutput(tf.keras.layers.Layer): + """ + The residual connection is defined in TFViTMAELayer instead of here (as is the case with other models), due to the + layernorm applied before each block. + """ + + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.dense = tf.keras.layers.Dense( + units=config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense" + ) + self.dropout = tf.keras.layers.Dropout(rate=config.hidden_dropout_prob) + + def call(self, hidden_states: tf.Tensor, input_tensor: tf.Tensor, training: bool = False) -> tf.Tensor: + hidden_states = self.dense(inputs=hidden_states) + hidden_states = self.dropout(inputs=hidden_states, training=training) + + return hidden_states + + +# Copied from transformers.models.vit.modeling_tf_vit.TFViTAttention with ViT->ViTMAE +class TFViTMAEAttention(tf.keras.layers.Layer): + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.self_attention = TFViTMAESelfAttention(config, name="attention") + self.dense_output = TFViTMAESelfOutput(config, name="output") + + def prune_heads(self, heads): + raise NotImplementedError + + def call( + self, + input_tensor: tf.Tensor, + head_mask: tf.Tensor, + output_attentions: bool, + training: bool = False, + ) -> Tuple[tf.Tensor]: + self_outputs = self.self_attention( + hidden_states=input_tensor, head_mask=head_mask, output_attentions=output_attentions, training=training + ) + attention_output = self.dense_output( + hidden_states=self_outputs[0], input_tensor=input_tensor, training=training + ) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + + return outputs + + +# Copied from transformers.models.vit.modeling_tf_vit.TFViTIntermediate with ViT->ViTMAE +class TFViTMAEIntermediate(tf.keras.layers.Layer): + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.dense = tf.keras.layers.Dense( + units=config.intermediate_size, kernel_initializer=get_initializer(config.initializer_range), name="dense" + ) + + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = get_tf_activation(config.hidden_act) + else: + self.intermediate_act_fn = config.hidden_act + + def call(self, hidden_states: tf.Tensor) -> tf.Tensor: + hidden_states = self.dense(inputs=hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + + return hidden_states + + +# Copied from transformers.models.vit.modeling_tf_vit.TFViTOutput with ViT->ViTMAE +class TFViTMAEOutput(tf.keras.layers.Layer): + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.dense = tf.keras.layers.Dense( + units=config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense" + ) + self.dropout = tf.keras.layers.Dropout(rate=config.hidden_dropout_prob) + + def call(self, hidden_states: tf.Tensor, input_tensor: tf.Tensor, training: bool = False) -> tf.Tensor: + hidden_states = self.dense(inputs=hidden_states) + hidden_states = self.dropout(inputs=hidden_states, training=training) + hidden_states = hidden_states + input_tensor + + return hidden_states + + +# Copied from transformers.models.vit.modeling_tf_vit.TFViTLayer with ViT->ViTMAE +class TFViTMAELayer(tf.keras.layers.Layer): + """This corresponds to the Block class in the timm implementation.""" + + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.attention = TFViTMAEAttention(config, name="attention") + self.intermediate = TFViTMAEIntermediate(config, name="intermediate") + self.vit_output = TFViTMAEOutput(config, name="output") + + self.layernorm_before = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name="layernorm_before" + ) + self.layernorm_after = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name="layernorm_after" + ) + + def call( + self, + hidden_states: tf.Tensor, + head_mask: tf.Tensor, + output_attentions: bool, + training: bool = False, + ) -> Tuple[tf.Tensor]: + attention_outputs = self.attention( + # in ViTMAE, layernorm is applied before self-attention + input_tensor=self.layernorm_before(inputs=hidden_states), + head_mask=head_mask, + output_attentions=output_attentions, + training=training, + ) + attention_output = attention_outputs[0] + + # first residual connection + hidden_states = attention_output + hidden_states + + # in ViTMAE, layernorm is also applied after self-attention + layer_output = self.layernorm_after(inputs=hidden_states) + + intermediate_output = self.intermediate(hidden_states=layer_output) + + # second residual connection is done here + layer_output = self.vit_output( + hidden_states=intermediate_output, input_tensor=hidden_states, training=training + ) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + + return outputs + + +# Copied from transformers.models.vit.modeling_tf_vit.TFViTEncoder with ViT->ViTMAE +class TFViTMAEEncoder(tf.keras.layers.Layer): + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.layer = [TFViTMAELayer(config, name=f"layer_._{i}") for i in range(config.num_hidden_layers)] + + def call( + self, + hidden_states: tf.Tensor, + head_mask: tf.Tensor, + output_attentions: bool, + output_hidden_states: bool, + return_dict: bool, + training: bool = False, + ) -> Union[TFBaseModelOutput, Tuple[tf.Tensor]]: + all_hidden_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module( + hidden_states=hidden_states, + head_mask=head_mask[i], + output_attentions=output_attentions, + training=training, + ) + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + # Add last layer + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, all_hidden_states, all_attentions] if v is not None) + + return TFBaseModelOutput( + last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_attentions + ) + + +@keras_serializable +class TFViTMAEMainLayer(tf.keras.layers.Layer): + config_class = ViTMAEConfig + + def __init__(self, config: ViTMAEConfig, **kwargs): + super().__init__(**kwargs) + + self.config = config + + self.embeddings = TFViTMAEEmbeddings(config, name="embeddings") + self.encoder = TFViTMAEEncoder(config, name="encoder") + self.layernorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="layernorm") + + def get_input_embeddings(self) -> tf.keras.layers.Layer: + return self.embeddings.patch_embeddings + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + raise NotImplementedError + + @unpack_inputs + def call( + self, + pixel_values: Optional[TFModelInputType] = None, + noise: tf.Tensor = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: bool = False, + **kwargs, + ) -> Union[TFViTMAEModelOutput, Tuple[tf.Tensor]]: + embedding_output, mask, ids_restore = self.embeddings( + pixel_values=pixel_values, training=training, noise=noise + ) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + raise NotImplementedError + else: + head_mask = [None] * self.config.num_hidden_layers + + encoder_outputs = self.encoder( + embedding_output, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + + sequence_output = encoder_outputs[0] + sequence_output = self.layernorm(inputs=sequence_output) + + if not return_dict: + return (sequence_output, mask, ids_restore) + encoder_outputs[1:] + + return TFViTMAEModelOutput( + last_hidden_state=sequence_output, + mask=mask, + ids_restore=ids_restore, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +class TFViTMAEPreTrainedModel(TFPreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = ViTMAEConfig + base_model_prefix = "vit" + main_input_name = "pixel_values" + + @property + def dummy_inputs(self) -> Dict[str, tf.Tensor]: + """ + Dummy inputs to build the network. Returns: + `Dict[str, tf.Tensor]`: The dummy inputs. + """ + VISION_DUMMY_INPUTS = tf.random.uniform( + shape=(3, self.config.num_channels, self.config.image_size, self.config.image_size), + dtype=tf.float32, + ) + return {"pixel_values": tf.constant(VISION_DUMMY_INPUTS)} + + @tf.function( + input_signature=[ + { + "pixel_values": tf.TensorSpec((None, None, None, None), tf.float32, name="pixel_values"), + } + ] + ) + def serving(self, inputs): + """ + Method used for serving the model. + + Args: + inputs (`Dict[str, tf.Tensor]`): + The input of the saved model as a dictionary of tensors. + """ + + return self.call(inputs) + + +VIT_MAE_START_DOCSTRING = r""" + This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it + as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and + behavior. + + + + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the + tensors in the first argument of the model call function: `model(inputs)`. + + + + Args: + config ([`ViTMAEConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. +""" + +VIT_MAE_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`AutoFeatureExtractor`]. See + [`AutoFeatureExtractor.__call__`] for details. + + head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the + config will be used instead. + + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. This argument can be used only in eager mode, in graph mode the value in the config will be + used instead. + + return_dict (`bool`, *optional*): + Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used + in eager mode, in graph mode the value will always be set to True. + + training (`bool`, *optional*, defaults to `False``): + Whether or not to use the model in training mode (some modules like dropout modules have different + behaviors between training and evaluation). +""" + + +@add_start_docstrings( + "The bare ViTMAE Model transformer outputting raw hidden-states without any specific head on top.", + VIT_MAE_START_DOCSTRING, +) +class TFViTMAEModel(TFViTMAEPreTrainedModel): + def __init__(self, config: ViTMAEConfig, *inputs, **kwargs): + super().__init__(config, *inputs, **kwargs) + + self.vit = TFViTMAEMainLayer(config, name="vit") + + def get_input_embeddings(self): + return self.vit.get_input_embeddings() + + @unpack_inputs + @add_start_docstrings_to_model_forward(VIT_MAE_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=TFViTMAEModelOutput, config_class=_CONFIG_FOR_DOC) + def call( + self, + pixel_values: Optional[TFModelInputType] = None, + noise: tf.Tensor = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: bool = False, + **kwargs, + ) -> Union[TFViTMAEModelOutput, Tuple[tf.Tensor]]: + r""" + Returns: + + Examples: + + ```python + >>> from transformers import AutoFeatureExtractor, TFViTMAEModel + >>> from PIL import Image + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/vit-mae-base") + >>> model = TFViTMAEModel.from_pretrained("facebook/vit-mae-base") + + >>> inputs = feature_extractor(images=image, return_tensors="tf") + >>> outputs = model(**inputs) + >>> last_hidden_states = outputs.last_hidden_state + ```""" + outputs = self.vit( + pixel_values=pixel_values, + noise=noise, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + + return outputs + + +class TFViTMAEDecoder(tf.keras.layers.Layer): + def __init__(self, config, num_patches, **kwargs): + super().__init__(**kwargs) + self.decoder_embed = tf.keras.layers.Dense(config.decoder_hidden_size, name="decoder_embed") + + decoder_config = deepcopy(config) + decoder_config.hidden_size = config.decoder_hidden_size + decoder_config.num_hidden_layers = config.decoder_num_hidden_layers + decoder_config.num_attention_heads = config.decoder_num_attention_heads + decoder_config.intermediate_size = config.decoder_intermediate_size + self.decoder_layers = [ + TFViTMAELayer(decoder_config, name=f"decoder_layers.{j}") for j in range(config.decoder_num_hidden_layers) + ] + + self.decoder_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="decoder_norm") + self.decoder_pred = tf.keras.layers.Dense( + config.patch_size**2 * config.num_channels, name="decoder_pred" + ) # encoder to decoder + self.config = config + self.num_patches = num_patches + + def build(self, input_shape: tf.TensorShape): + self.mask_token = self.add_weight( + shape=(1, 1, self.config.decoder_hidden_size), + initializer=tf.random_normal_initializer(stddev=self.config.initializer_range), + trainable=True, + name="mask_token", + ) + self.decoder_pos_embed = self.add_weight( + shape=(1, self.num_patches + 1, self.config.decoder_hidden_size), + initializer="zeros", + trainable=False, + name="decoder_pos_embed", + ) + decoder_pos_embed = get_2d_sincos_pos_embed( + self.decoder_pos_embed.shape[-1], + int(self.num_patches**0.5), + add_cls_token=True, + )[None, ...] + self.decoder_pos_embed.assign(decoder_pos_embed) + + super().build(input_shape) + + def call( + self, + hidden_states, + ids_restore, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + # embed tokens + x = self.decoder_embed(hidden_states) + + # append mask tokens to sequence + mask_tokens = tf.tile( + self.mask_token, + (shape_list(x)[0], shape_list(ids_restore)[1] + 1 - shape_list(x)[1], 1), + ) + x_ = tf.concat([x[:, 1:, :], mask_tokens], axis=1) # no cls token + x_ = tf.gather(x_, axis=1, batch_dims=1, indices=ids_restore) # unshuffle + x = tf.concat([x[:, :1, :], x_], axis=1) # append cls token + + # add pos embed + hidden_states = x + self.decoder_pos_embed + + # apply Transformer layers (blocks) + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + for i, layer_module in enumerate(self.decoder_layers): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module( + hidden_states, + head_mask=None, + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + hidden_states = self.decoder_norm(hidden_states) + + # predictor projection + logits = self.decoder_pred(hidden_states) + + # remove cls token + logits = logits[:, 1:, :] + + if not return_dict: + return tuple(v for v in [logits, all_hidden_states, all_self_attentions] if v is not None) + return TFViTMAEDecoderOutput(logits=logits, hidden_states=all_hidden_states, attentions=all_self_attentions) + + +@add_start_docstrings( + "The ViTMAE Model transformer with the decoder on top for self-supervised pre-training.", + VIT_MAE_START_DOCSTRING, +) +class TFViTMAEForPreTraining(TFViTMAEPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.config = config + + self.vit = TFViTMAEMainLayer(config, name="vit") + self.decoder = TFViTMAEDecoder( + config, + num_patches=self.vit.embeddings.num_patches, + name="decoder", + ) + + def get_input_embeddings(self): + return self.vit.get_input_embeddings() + + def _prune_heads(self, heads_to_prune): + raise NotImplementedError + + def patchify(self, imgs): + """ + imgs: (batch_size, height, width, 3) x: (batch_size, num_patches, patch_size**2 *3) + """ + imgs = tf.cond( + tf.math.equal(shape_list(imgs)[1], 3), lambda: tf.transpose(imgs, perm=(0, 2, 3, 1)), lambda: imgs + ) + + p = self.vit.embeddings.patch_embeddings.patch_size[0] + tf.debugging.assert_equal(shape_list(imgs)[1], shape_list(imgs)[2]) + tf.debugging.assert_equal(shape_list(imgs)[1] % p, 0) + + h = w = shape_list(imgs)[2] // p + x = tf.reshape(imgs, (shape_list(imgs)[0], h, p, w, p, 3)) + x = tf.einsum("nhpwqc->nhwpqc", x) + x = tf.reshape(x, (shape_list(imgs)[0], h * w, p**2 * 3)) + return x + + def unpatchify(self, x): + """ + x: (batch_size, num_patches, patch_size**2 *3) imgs: (batch_size, height, width, 3) + """ + p = self.vit.embeddings.patch_embeddings.patch_size[0] + h = w = int(shape_list(x)[1] ** 0.5) + tf.debugging.assert_equal(h * w, shape_list(x)[1]) + + x = tf.reshape(x, (shape_list(x)[0], h, w, p, p, 3)) + x = tf.einsum("nhwpqc->nhpwqc", x) + imgs = tf.reshape(x, (shape_list(x)[0], h * p, h * p, 3)) + return imgs + + def forward_loss(self, imgs, pred, mask): + """ + imgs: [batch_size, height, width, 3] pred: [batch_size, num_patches, patch_size**2*3] mask: [N, L], 0 is keep, + 1 is remove, + """ + target = self.patchify(imgs) + if self.config.norm_pix_loss: + mean = tf.reduce_mean(target, axis=-1, keepdims=True) + var = tf.math.reduce_variance(target, axis=-1, keepdims=True) + target = (target - mean) / (var + 1.0e-6) ** 0.5 + + loss = (pred - target) ** 2 + loss = tf.reduce_mean(loss, axis=-1) # [N, L], mean loss per patch + + loss = tf.reduce_sum(loss * mask) / tf.reduce_sum(mask) # mean loss on removed patches + return loss + + @unpack_inputs + @add_start_docstrings_to_model_forward(VIT_MAE_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=TFViTMAEForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) + def call( + self, + pixel_values: Optional[TFModelInputType] = None, + noise: tf.Tensor = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: bool = False, + **kwargs, + ) -> Union[TFViTMAEForPreTrainingOutput, Tuple[tf.Tensor]]: + r""" + Returns: + + Examples: + + ```python + >>> from transformers import AutoFeatureExtractor, TFViTMAEForPreTraining + >>> from PIL import Image + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/vit-mae-base") + >>> model = TFViTMAEForPreTraining.from_pretrained("facebook/vit-mae-base") + + >>> inputs = feature_extractor(images=image, return_tensors="pt") + >>> outputs = model(**inputs) + >>> loss = outputs.loss + >>> mask = outputs.mask + >>> ids_restore = outputs.ids_restore + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.vit( + pixel_values=pixel_values, + noise=noise, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + + latent = outputs.last_hidden_state + ids_restore = outputs.ids_restore + mask = outputs.mask + + decoder_outputs = self.decoder(latent, ids_restore) # [batch_size, num_patches, patch_size**2*3] + logits = decoder_outputs.logits + + loss = self.forward_loss(pixel_values, logits, mask) + + if not return_dict: + output = (logits, mask, ids_restore) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TFViTMAEForPreTrainingOutput( + loss=loss, + logits=logits, + mask=mask, + ids_restore=ids_restore, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/src/transformers/models/vit_mae/modeling_vit_mae.py b/src/transformers/models/vit_mae/modeling_vit_mae.py index d51e47e63bb2..8acae9fca754 100755 --- a/src/transformers/models/vit_mae/modeling_vit_mae.py +++ b/src/transformers/models/vit_mae/modeling_vit_mae.py @@ -240,18 +240,21 @@ def initialize_weights(self): # timm's trunc_normal_(std=.02) is effectively normal_(std=0.02) as cutoff is too big (2.) torch.nn.init.normal_(self.cls_token, std=self.config.initializer_range) - def random_masking(self, sequence): + def random_masking(self, sequence, noise=None): """ Perform per-sample random masking by per-sample shuffling. Per-sample shuffling is done by argsort random noise. Args: sequence (`torch.LongTensor` of shape `(batch_size, sequence_length, dim)`) + noise (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*) which is + mainly used for testing purposes to control randomness and maintain the reproducibility """ batch_size, seq_length, dim = sequence.shape len_keep = int(seq_length * (1 - self.config.mask_ratio)) - noise = torch.rand(batch_size, seq_length, device=sequence.device) # noise in [0, 1] + if noise is None: + noise = torch.rand(batch_size, seq_length, device=sequence.device) # noise in [0, 1] # sort noise for each sample ids_shuffle = torch.argsort(noise, dim=1) # ascend: small is keep, large is remove @@ -269,7 +272,7 @@ def random_masking(self, sequence): return sequence_masked, mask, ids_restore - def forward(self, pixel_values): + def forward(self, pixel_values, noise=None): batch_size, num_channels, height, width = pixel_values.shape embeddings = self.patch_embeddings(pixel_values) @@ -277,7 +280,7 @@ def forward(self, pixel_values): embeddings = embeddings + self.position_embeddings[:, 1:, :] # masking: length -> length * config.mask_ratio - embeddings, mask, ids_restore = self.random_masking(embeddings) + embeddings, mask, ids_restore = self.random_masking(embeddings, noise) # append cls token cls_token = self.cls_token + self.position_embeddings[:, :1, :] @@ -668,6 +671,7 @@ class PreTrainedModel def forward( self, pixel_values=None, + noise=None, head_mask=None, output_attentions=None, output_hidden_states=None, @@ -709,7 +713,7 @@ def forward( # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) - embedding_output, mask, ids_restore = self.embeddings(pixel_values) + embedding_output, mask, ids_restore = self.embeddings(pixel_values, noise=noise) encoder_outputs = self.encoder( embedding_output, @@ -910,6 +914,7 @@ def forward_loss(self, imgs, pred, mask): def forward( self, pixel_values=None, + noise=None, head_mask=None, output_attentions=None, output_hidden_states=None, @@ -941,6 +946,7 @@ def forward( outputs = self.vit( pixel_values, + noise=noise, head_mask=head_mask, output_attentions=output_attentions, output_hidden_states=output_hidden_states, diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index 202cd481ebeb..db6d8d467aa2 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -1987,6 +1987,27 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) +class TFViTMAEForPreTraining(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFViTMAEModel(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFViTMAEPreTrainedModel(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/tests/vit_mae/test_modeling_tf_vit_mae.py b/tests/vit_mae/test_modeling_tf_vit_mae.py new file mode 100644 index 000000000000..c0fb84856934 --- /dev/null +++ b/tests/vit_mae/test_modeling_tf_vit_mae.py @@ -0,0 +1,718 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the TensorFlow ViTMAE model. """ + + +import copy +import inspect +import json +import math +import os +import tempfile +import unittest +from importlib import import_module + +import numpy as np + +from transformers import ViTMAEConfig +from transformers.file_utils import cached_property, is_tf_available, is_vision_available +from transformers.testing_utils import is_pt_tf_cross_test, require_tf, require_vision, slow, torch_device + +from ..test_configuration_common import ConfigTester +from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor + + +if is_tf_available(): + import tensorflow as tf + + from transformers import TFViTMAEForPreTraining, TFViTMAEModel + from transformers.models.vit_mae.modeling_tf_vit_mae import to_2tuple + + +if is_vision_available(): + from PIL import Image + + from transformers import ViTFeatureExtractor + + +class TFViTMAEModelTester: + def __init__( + self, + parent, + batch_size=13, + image_size=30, + patch_size=2, + num_channels=3, + is_training=True, + use_labels=True, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + type_sequence_label_size=10, + initializer_range=0.02, + num_labels=3, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.image_size = image_size + self.patch_size = patch_size + self.num_channels = num_channels + self.is_training = is_training + self.use_labels = use_labels + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.scope = scope + + def prepare_config_and_inputs(self): + pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) + + labels = None + if self.use_labels: + labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + + config = self.get_config() + + return config, pixel_values, labels + + def get_config(self): + return ViTMAEConfig( + image_size=self.image_size, + patch_size=self.patch_size, + num_channels=self.num_channels, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + is_decoder=False, + initializer_range=self.initializer_range, + ) + + def create_and_check_model(self, config, pixel_values, labels): + model = TFViTMAEModel(config=config) + result = model(pixel_values, training=False) + # expected sequence length = (num_patches + 1) * (1 - config.mask_ratio), rounded above + # (we add 1 for the [CLS] token) + image_size = to_2tuple(self.image_size) + patch_size = to_2tuple(self.patch_size) + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + expected_seq_len = int(math.ceil((1 - config.mask_ratio) * (num_patches + 1))) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, expected_seq_len, self.hidden_size)) + + def create_and_check_for_pretraining(self, config, pixel_values, labels): + model = TFViTMAEForPreTraining(config) + result = model(pixel_values, training=False) + # expected sequence length = num_patches + image_size = to_2tuple(self.image_size) + patch_size = to_2tuple(self.patch_size) + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + expected_seq_len = num_patches + expected_num_channels = self.patch_size**2 * self.num_channels + self.parent.assertEqual(result.logits.shape, (self.batch_size, expected_seq_len, expected_num_channels)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, pixel_values, labels) = config_and_inputs + inputs_dict = {"pixel_values": pixel_values} + return config, inputs_dict + + +@require_tf +class TFViTMAEModelTest(TFModelTesterMixin, unittest.TestCase): + """ + Here we also overwrite some of the tests of test_modeling_common.py, as ViTMAE does not use input_ids, inputs_embeds, + attention_mask and seq_length. + """ + + all_model_classes = (TFViTMAEModel, TFViTMAEForPreTraining) if is_tf_available() else () + + test_pruning = False + test_onnx = False + test_resize_embeddings = False + test_head_masking = False + + def setUp(self): + self.model_tester = TFViTMAEModelTester(self) + self.config_tester = ConfigTester(self, config_class=ViTMAEConfig, has_text_modality=False, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + @unittest.skip(reason="ViTMAE does not use inputs_embeds") + def test_inputs_embeds(self): + # ViTMAE does not use inputs_embeds + pass + + def test_model_common_attributes(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + self.assertIsInstance(model.get_input_embeddings(), (tf.keras.layers.Layer)) + x = model.get_output_embeddings() + self.assertTrue(x is None or isinstance(x, tf.keras.layers.Layer)) + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.call) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["pixel_values"] + self.assertListEqual(arg_names[:1], expected_arg_names) + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_for_pretraining(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_pretraining(*config_and_inputs) + + # overwrite from common since TFViTMAEForPretraining has random masking, we need to fix the noise + # to generate masks during test + def test_keyword_and_dict_args(self): + # make the mask reproducible + np.random.seed(2) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + num_patches = int((config.image_size // config.patch_size) ** 2) + noise = np.random.uniform(size=(self.model_tester.batch_size, num_patches)) + + for model_class in self.all_model_classes: + model = model_class(config) + inputs = self._prepare_for_class(inputs_dict, model_class) + + outputs_dict = model(inputs, noise=noise) + + inputs_keywords = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) + outputs_keywords = model(**inputs_keywords, noise=noise) + output_dict = outputs_dict[0].numpy() + output_keywords = outputs_keywords[0].numpy() + + self.assertLess(np.sum(np.abs(output_dict - output_keywords)), 1e-6) + + # overwrite from common since TFViTMAEForPretraining has random masking, we need to fix the noise + # to generate masks during test + def test_numpy_arrays_inputs(self): + # make the mask reproducible + np.random.seed(2) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + num_patches = int((config.image_size // config.patch_size) ** 2) + noise = np.random.uniform(size=(self.model_tester.batch_size, num_patches)) + + def prepare_numpy_arrays(inputs_dict): + inputs_np_dict = {} + for k, v in inputs_dict.items(): + if tf.is_tensor(v): + inputs_np_dict[k] = v.numpy() + else: + inputs_np_dict[k] = np.array(k) + + return inputs_np_dict + + for model_class in self.all_model_classes: + model = model_class(config) + + inputs = self._prepare_for_class(inputs_dict, model_class) + inputs_np = prepare_numpy_arrays(inputs) + + output_for_dict_input = model(inputs_np, noise=noise) + output_for_kw_input = model(**inputs_np, noise=noise) + self.assert_outputs_same(output_for_dict_input, output_for_kw_input) + + def test_attention_outputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.return_dict = True + + # in ViTMAE, the seq_len equals (number of patches + 1) * (1 - mask_ratio), rounded above + image_size = to_2tuple(self.model_tester.image_size) + patch_size = to_2tuple(self.model_tester.patch_size) + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + seq_len = int(math.ceil((1 - config.mask_ratio) * (num_patches + 1))) + encoder_seq_length = getattr(self.model_tester, "encoder_seq_length", seq_len) + encoder_key_length = getattr(self.model_tester, "key_length", encoder_seq_length) + chunk_length = getattr(self.model_tester, "chunk_length", None) + if chunk_length is not None and hasattr(self.model_tester, "num_hashes"): + encoder_seq_length = encoder_seq_length * self.model_tester.num_hashes + + for model_class in self.all_model_classes: + inputs_dict["output_attentions"] = True + inputs_dict["output_hidden_states"] = False + config.return_dict = True + model = model_class(config) + outputs = model(**self._prepare_for_class(inputs_dict, model_class), training=False) + attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions + self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + + # check that output_attentions also work using config + del inputs_dict["output_attentions"] + config.output_attentions = True + model = model_class(config) + outputs = model(**self._prepare_for_class(inputs_dict, model_class), training=False) + attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions + self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + + if chunk_length is not None: + self.assertListEqual( + list(attentions[0].shape[-4:]), + [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length], + ) + else: + self.assertListEqual( + list(attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length], + ) + out_len = len(outputs) + + # Check attention is always last and order is fine + inputs_dict["output_attentions"] = True + inputs_dict["output_hidden_states"] = True + model = model_class(config) + outputs = model(**self._prepare_for_class(inputs_dict, model_class), training=False) + + if hasattr(self.model_tester, "num_hidden_states_types"): + added_hidden_states = self.model_tester.num_hidden_states_types + elif self.is_encoder_decoder: + added_hidden_states = 2 + else: + added_hidden_states = 1 + self.assertEqual(out_len + added_hidden_states, len(outputs)) + + self_attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions + + self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers) + if chunk_length is not None: + self.assertListEqual( + list(self_attentions[0].shape[-4:]), + [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length], + ) + else: + self.assertListEqual( + list(self_attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length], + ) + + def test_hidden_states_output(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(config) + + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + + hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + + expected_num_layers = getattr( + self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1 + ) + self.assertEqual(len(hidden_states), expected_num_layers) + + # ViTMAE has a different seq_length + image_size = to_2tuple(self.model_tester.image_size) + patch_size = to_2tuple(self.model_tester.patch_size) + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + seq_length = int(math.ceil((1 - config.mask_ratio) * (num_patches + 1))) + + self.assertListEqual( + list(hidden_states[0].shape[-2:]), + [seq_length, self.model_tester.hidden_size], + ) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + # overwrite from common since TFViTMAEForPretraining has random masking, we need to fix the noise + # to generate masks during test + @is_pt_tf_cross_test + def test_pt_tf_model_equivalence(self): + import torch + + import transformers + + # make masks reproducible + np.random.seed(2) + + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + num_patches = int((config.image_size // config.patch_size) ** 2) + noise = np.random.uniform(size=(self.model_tester.batch_size, num_patches)) + pt_noise = torch.from_numpy(noise).to(device=torch_device) + tf_noise = tf.constant(noise) + + def prepare_pt_inputs_from_tf_inputs(tf_inputs_dict): + + pt_inputs_dict = {} + for name, key in tf_inputs_dict.items(): + pt_inputs_dict[name] = torch.from_numpy(key.numpy()).to(torch.float32) + + return pt_inputs_dict + + def check_outputs(tf_outputs, pt_outputs, model_class, names): + """ + Args: + model_class: The class of the model that is currently testing. For example, `TFBertModel`, + TFBertForMaskedLM`, `TFBertForSequenceClassification`, etc. Currently unused, but it could make + debugging easier and faster. + + names: A string, or a tuple of strings. These specify what tf_outputs/pt_outputs represent in the model outputs. + Currently unused, but in the future, we could use this information to make the error message clearer + by giving the name(s) of the output tensor(s) with large difference(s) between PT and TF. + """ + + # Allow `list` because `(TF)TransfoXLModelOutput.mems` is a list of tensors. + if type(tf_outputs) in [tuple, list]: + self.assertEqual(type(tf_outputs), type(pt_outputs)) + self.assertEqual(len(tf_outputs), len(pt_outputs)) + if type(names) == tuple: + for tf_output, pt_output, name in zip(tf_outputs, pt_outputs, names): + check_outputs(tf_output, pt_output, model_class, names=name) + elif type(names) == str: + for idx, (tf_output, pt_output) in enumerate(zip(tf_outputs, pt_outputs)): + check_outputs(tf_output, pt_output, model_class, names=f"{names}_{idx}") + else: + raise ValueError(f"`names` should be a `tuple` or a string. Got {type(names)} instead.") + elif isinstance(tf_outputs, tf.Tensor): + self.assertTrue(isinstance(pt_outputs, torch.Tensor)) + + tf_outputs = tf_outputs.numpy() + if isinstance(tf_outputs, np.float32): + tf_outputs = np.array(tf_outputs, dtype=np.float32) + pt_outputs = pt_outputs.detach().to("cpu").numpy() + + tf_nans = np.isnan(tf_outputs) + pt_nans = np.isnan(pt_outputs) + + pt_outputs[tf_nans] = 0 + tf_outputs[tf_nans] = 0 + pt_outputs[pt_nans] = 0 + tf_outputs[pt_nans] = 0 + + max_diff = np.amax(np.abs(tf_outputs - pt_outputs)) + self.assertLessEqual(max_diff, 1e-5) + else: + raise ValueError( + f"`tf_outputs` should be a `tuple` or an instance of `tf.Tensor`. Got {type(tf_outputs)} instead." + ) + + def check_pt_tf_models(tf_model, pt_model): + # we are not preparing a model with labels because of the formation + # of the ViT MAE model + + # send pytorch model to the correct device + pt_model.to(torch_device) + + # Check predictions on first output (logits/hidden-states) are close enough given low-level computational differences + pt_model.eval() + + pt_inputs_dict = prepare_pt_inputs_from_tf_inputs(tf_inputs_dict) + + # send pytorch inputs to the correct device + pt_inputs_dict = { + k: v.to(device=torch_device) if isinstance(v, torch.Tensor) else v for k, v in pt_inputs_dict.items() + } + + # Original test: check without `labels` + with torch.no_grad(): + pt_outputs = pt_model(**pt_inputs_dict, noise=pt_noise) + tf_outputs = tf_model(tf_inputs_dict, noise=tf_noise) + + tf_keys = tuple([k for k, v in tf_outputs.items() if v is not None]) + pt_keys = tuple([k for k, v in pt_outputs.items() if v is not None]) + + self.assertEqual(tf_keys, pt_keys) + check_outputs(tf_outputs.to_tuple(), pt_outputs.to_tuple(), model_class, names=tf_keys) + + for model_class in self.all_model_classes: + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # Output all for aggressive testing + config.output_hidden_states = True + if self.has_attentions: + config.output_attentions = True + + pt_model_class_name = model_class.__name__[2:] # Skip the "TF" at the beginning + pt_model_class = getattr(transformers, pt_model_class_name) + + tf_model = model_class(config) + pt_model = pt_model_class(config) + + tf_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + + # Check we can load pt model in tf and vice-versa with model => model functions + tf_model = transformers.load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=tf_inputs_dict) + pt_model = transformers.load_tf2_model_in_pytorch_model(pt_model, tf_model) + + check_pt_tf_models(tf_model, pt_model) + + # Check we can load pt model in tf and vice-versa with checkpoint => model functions + with tempfile.TemporaryDirectory() as tmpdirname: + pt_checkpoint_path = os.path.join(tmpdirname, "pt_model.bin") + torch.save(pt_model.state_dict(), pt_checkpoint_path) + tf_model = transformers.load_pytorch_checkpoint_in_tf2_model(tf_model, pt_checkpoint_path) + + tf_checkpoint_path = os.path.join(tmpdirname, "tf_model.h5") + tf_model.save_weights(tf_checkpoint_path) + pt_model = transformers.load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path) + + check_pt_tf_models(tf_model, pt_model) + + # overwrite from common since TFViTMAEForPretraining outputs loss along with + # logits and mask indices. loss and mask indicies are not suitable for integration + # with other keras modules. + def test_compile_tf_model(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0) + loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy") + + for model_class in self.all_model_classes: + # `pixel_values` implies that the input is an image + inputs = tf.keras.Input( + batch_shape=( + 3, + self.model_tester.num_channels, + self.model_tester.image_size, + self.model_tester.image_size, + ), + name="pixel_values", + dtype="float32", + ) + + # Prepare our model + model = model_class(config) + model(self._prepare_for_class(inputs_dict, model_class)) # Model must be called before saving. + # Let's load it from the disk to be sure we can use pretrained weights + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, saved_model=False) + model = model_class.from_pretrained(tmpdirname) + + outputs_dict = model(inputs) + hidden_states = outputs_dict[0] + + # `TFViTMAEForPreTraining` outputs are not recommended to be used for + # downstream application. This is just to check if the outputs of + # `TFViTMAEForPreTraining` can be integrated with other keras modules. + if model_class.__name__ == "TFViTMAEForPreTraining": + hidden_states = outputs_dict["logits"] + + # Add a dense layer on top to test integration with other keras modules + outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states) + + # Compile extended model + extended_model = tf.keras.Model(inputs=[inputs], outputs=[outputs]) + extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + + # overwrite from common since TFViTMAEForPretraining has random masking, we need to fix the noise + # to generate masks during test + def test_keras_save_load(self): + # make mask reproducible + np.random.seed(2) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + tf_main_layer_classes = set( + module_member + for model_class in self.all_model_classes + for module in (import_module(model_class.__module__),) + for module_member_name in dir(module) + if module_member_name.endswith("MainLayer") + # This condition is required, since `modeling_tf_clip.py` has 3 classes whose names end with `MainLayer`. + and module_member_name[: -len("MainLayer")] == model_class.__name__[: -len("Model")] + for module_member in (getattr(module, module_member_name),) + if isinstance(module_member, type) + and tf.keras.layers.Layer in module_member.__bases__ + and getattr(module_member, "_keras_serializable", False) + ) + + num_patches = int((config.image_size // config.patch_size) ** 2) + noise = np.random.uniform(size=(self.model_tester.batch_size, num_patches)) + noise = tf.convert_to_tensor(noise) + inputs_dict.update({"noise": noise}) + + for main_layer_class in tf_main_layer_classes: + main_layer = main_layer_class(config) + + symbolic_inputs = { + name: tf.keras.Input(tensor.shape[1:], dtype=tensor.dtype) for name, tensor in inputs_dict.items() + } + + model = tf.keras.Model(symbolic_inputs, outputs=main_layer(symbolic_inputs)) + outputs = model(inputs_dict) + + with tempfile.TemporaryDirectory() as tmpdirname: + filepath = os.path.join(tmpdirname, "keras_model.h5") + model.save(filepath) + model = tf.keras.models.load_model( + filepath, custom_objects={main_layer_class.__name__: main_layer_class} + ) + assert isinstance(model, tf.keras.Model) + after_outputs = model(inputs_dict) + self.assert_outputs_same(after_outputs, outputs) + + # overwrite from common since TFViTMAEForPretraining has random masking, we need to fix the noise + # to generate masks during test + def test_save_load(self): + # make mask reproducible + np.random.seed(2) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + num_patches = int((config.image_size // config.patch_size) ** 2) + noise = np.random.uniform(size=(self.model_tester.batch_size, num_patches)) + + for model_class in self.all_model_classes: + model = model_class(config) + model_input = self._prepare_for_class(inputs_dict, model_class) + outputs = model(model_input, noise=noise) + + if model_class.__name__ == "TFViTMAEModel": + out_2 = outputs.last_hidden_state.numpy() + out_2[np.isnan(out_2)] = 0 + else: + out_2 = outputs.logits.numpy() + out_2[np.isnan(out_2)] = 0 + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, saved_model=True) + saved_model_dir = os.path.join(tmpdirname, "saved_model", "1") + model = tf.keras.models.load_model(saved_model_dir) + after_outputs = model(model_input, noise=noise) + + if model_class.__name__ == "TFViTMAEModel": + out_1 = after_outputs["last_hidden_state"].numpy() + out_1[np.isnan(out_1)] = 0 + else: + out_1 = after_outputs["logits"].numpy() + out_1[np.isnan(out_1)] = 0 + + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, 1e-5) + + # overwrite from common since TFViTMAEForPretraining has random masking, we need to fix the noise + # to generate masks during test + def test_save_load_config(self): + # make mask reproducible + np.random.seed(2) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + num_patches = int((config.image_size // config.patch_size) ** 2) + noise = np.random.uniform(size=(self.model_tester.batch_size, num_patches)) + + for model_class in self.all_model_classes: + model = model_class(config) + model_inputs = self._prepare_for_class(inputs_dict, model_class) + + outputs = model(model_inputs, noise=noise) + model_config = model.get_config() + # make sure that returned config is jsonifiable, which is required by keras + json.dumps(model_config) + new_model = model_class.from_config(model.get_config()) + # make sure it also accepts a normal config + _ = model_class.from_config(model.config) + _ = new_model(model_inputs) # Build model + new_model.set_weights(model.get_weights()) + after_outputs = new_model(model_inputs, noise=noise) + + self.assert_outputs_same(after_outputs, outputs) + + @unittest.skip( + reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load + to get deterministic results.""" + ) + def test_determinism(self): + pass + + @unittest.skip(reason="""ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load""") + def test_model_outputs_equivalence(self): + pass + + @slow + def test_model_from_pretrained(self): + + model = TFViTMAEModel.from_pretrained("google/vit-base-patch16-224") + self.assertIsNotNone(model) + + +# We will verify our results on an image of cute cats +def prepare_img(): + image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") + return image + + +@require_tf +@require_vision +class TFViTMAEModelIntegrationTest(unittest.TestCase): + @cached_property + def default_feature_extractor(self): + return ViTFeatureExtractor.from_pretrained("facebook/vit-mae-base") if is_vision_available() else None + + @slow + def test_inference_for_pretraining(self): + # make random mask reproducible across the PT and TF model + np.random.seed(2) + + model = TFViTMAEForPreTraining.from_pretrained("facebook/vit-mae-base") + + feature_extractor = self.default_feature_extractor + image = prepare_img() + inputs = feature_extractor(images=image, return_tensors="tf") + + # prepare a noise vector that will be also used for testing the TF model + # (this way we can ensure that the PT and TF models operate on the same inputs) + vit_mae_config = ViTMAEConfig() + num_patches = int((vit_mae_config.image_size // vit_mae_config.patch_size) ** 2) + noise = np.random.uniform(size=(1, num_patches)) + + # forward pass + outputs = model(**inputs, noise=noise) + + # verify the logits + expected_shape = tf.convert_to_tensor([1, 196, 768]) + self.assertEqual(outputs.logits.shape, expected_shape) + + expected_slice = tf.convert_to_tensor( + [[-0.0548, -1.7023, -0.9325], [0.3721, -0.5670, -0.2233], [0.8235, -1.3878, -0.3524]] + ) + + tf.debugging.assert_near(outputs.logits[0, :3, :3], expected_slice, atol=1e-4) diff --git a/tests/vit_mae/test_modeling_vit_mae.py b/tests/vit_mae/test_modeling_vit_mae.py index 5efb22c3c314..6ae62cb1c2c3 100644 --- a/tests/vit_mae/test_modeling_vit_mae.py +++ b/tests/vit_mae/test_modeling_vit_mae.py @@ -17,13 +17,14 @@ import inspect import math +import os import tempfile import unittest import numpy as np from transformers import ViTMAEConfig -from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.testing_utils import is_pt_tf_cross_test, require_torch, require_vision, slow, torch_device from transformers.utils import cached_property, is_torch_available, is_vision_available from ..test_configuration_common import ConfigTester @@ -139,11 +140,7 @@ def create_and_check_for_pretraining(self, config, pixel_values, labels): def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() - ( - config, - pixel_values, - labels, - ) = config_and_inputs + config, pixel_values, labels = config_and_inputs inputs_dict = {"pixel_values": pixel_values} return config, inputs_dict @@ -322,6 +319,153 @@ def check_hidden_states_output(inputs_dict, config, model_class): check_hidden_states_output(inputs_dict, config, model_class) + # overwrite from common since ViTMAEForPretraining has random masking, we need to fix the noise + # to generate masks during test + @is_pt_tf_cross_test + def test_pt_tf_model_equivalence(self): + import numpy as np + import tensorflow as tf + + import transformers + + # make masks reproducible + np.random.seed(2) + + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + num_patches = int((config.image_size // config.patch_size) ** 2) + noise = np.random.uniform(size=(self.model_tester.batch_size, num_patches)) + pt_noise = torch.from_numpy(noise).to(device=torch_device) + tf_noise = tf.constant(noise) + + def prepare_tf_inputs_from_pt_inputs(pt_inputs_dict): + + tf_inputs_dict = {} + for key, tensor in pt_inputs_dict.items(): + tf_inputs_dict[key] = tf.convert_to_tensor(tensor.cpu().numpy(), dtype=tf.float32) + + return tf_inputs_dict + + def check_outputs(tf_outputs, pt_outputs, model_class, names): + """ + Args: + model_class: The class of the model that is currently testing. For example, `TFBertModel`, + TFBertForMaskedLM`, `TFBertForSequenceClassification`, etc. Currently unused, but it could make + debugging easier and faster. + + names: A string, or a tuple of strings. These specify what tf_outputs/pt_outputs represent in the model outputs. + Currently unused, but in the future, we could use this information to make the error message clearer + by giving the name(s) of the output tensor(s) with large difference(s) between PT and TF. + """ + + # Allow `list` because `(TF)TransfoXLModelOutput.mems` is a list of tensors. + if type(tf_outputs) in [tuple, list]: + self.assertEqual(type(tf_outputs), type(pt_outputs)) + self.assertEqual(len(tf_outputs), len(pt_outputs)) + if type(names) == tuple: + for tf_output, pt_output, name in zip(tf_outputs, pt_outputs, names): + check_outputs(tf_output, pt_output, model_class, names=name) + elif type(names) == str: + for idx, (tf_output, pt_output) in enumerate(zip(tf_outputs, pt_outputs)): + check_outputs(tf_output, pt_output, model_class, names=f"{names}_{idx}") + else: + raise ValueError(f"`names` should be a `tuple` or a string. Got {type(names)} instead.") + elif isinstance(tf_outputs, tf.Tensor): + self.assertTrue(isinstance(pt_outputs, torch.Tensor)) + + tf_outputs = tf_outputs.numpy() + if isinstance(tf_outputs, np.float32): + tf_outputs = np.array(tf_outputs, dtype=np.float32) + pt_outputs = pt_outputs.detach().to("cpu").numpy() + + tf_nans = np.isnan(tf_outputs) + pt_nans = np.isnan(pt_outputs) + + pt_outputs[tf_nans] = 0 + tf_outputs[tf_nans] = 0 + pt_outputs[pt_nans] = 0 + tf_outputs[pt_nans] = 0 + + max_diff = np.amax(np.abs(tf_outputs - pt_outputs)) + self.assertLessEqual(max_diff, 1e-5) + else: + raise ValueError( + f"`tf_outputs` should be a `tuple` or an instance of `tf.Tensor`. Got {type(tf_outputs)} instead." + ) + + def check_pt_tf_models(tf_model, pt_model, pt_inputs_dict): + # we are not preparing a model with labels because of the formation + # of the ViT MAE model + + # send pytorch model to the correct device + pt_model.to(torch_device) + + # Check predictions on first output (logits/hidden-states) are close enough given low-level computational differences + pt_model.eval() + + tf_inputs_dict = prepare_tf_inputs_from_pt_inputs(pt_inputs_dict) + + # send pytorch inputs to the correct device + pt_inputs_dict = { + k: v.to(device=torch_device) if isinstance(v, torch.Tensor) else v for k, v in pt_inputs_dict.items() + } + + # Original test: check without `labels` + with torch.no_grad(): + pt_outputs = pt_model(**pt_inputs_dict, noise=pt_noise) + tf_outputs = tf_model(tf_inputs_dict, noise=tf_noise) + + tf_keys = tuple([k for k, v in tf_outputs.items() if v is not None]) + pt_keys = tuple([k for k, v in pt_outputs.items() if v is not None]) + + self.assertEqual(tf_keys, pt_keys) + check_outputs(tf_outputs.to_tuple(), pt_outputs.to_tuple(), model_class, names=tf_keys) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + tf_model_class_name = "TF" + model_class.__name__ # Add the "TF" at the beginning + + # Output all for aggressive testing + config.output_hidden_states = True + config.output_attentions = self.has_attentions + + tf_model_class = getattr(transformers, tf_model_class_name) + + tf_model = tf_model_class(config) + pt_model = model_class(config) + + # make sure only tf inputs are forward that actually exist in function args + tf_input_keys = set(inspect.signature(tf_model.call).parameters.keys()) + + # remove all head masks + tf_input_keys.discard("head_mask") + tf_input_keys.discard("cross_attn_head_mask") + tf_input_keys.discard("decoder_head_mask") + + pt_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + + pt_inputs_dict = {k: v for k, v in pt_inputs_dict.items() if k in tf_input_keys} + + # Check we can load pt model in tf and vice-versa with model => model functions + tf_inputs_dict = prepare_tf_inputs_from_pt_inputs(pt_inputs_dict) + tf_model = transformers.load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=tf_inputs_dict) + pt_model = transformers.load_tf2_model_in_pytorch_model(pt_model, tf_model) + + check_pt_tf_models(tf_model, pt_model, pt_inputs_dict) + + # Check we can load pt model in tf and vice-versa with checkpoint => model functions + with tempfile.TemporaryDirectory() as tmpdirname: + pt_checkpoint_path = os.path.join(tmpdirname, "pt_model.bin") + torch.save(pt_model.state_dict(), pt_checkpoint_path) + tf_model = transformers.load_pytorch_checkpoint_in_tf2_model(tf_model, pt_checkpoint_path) + + tf_checkpoint_path = os.path.join(tmpdirname, "tf_model.h5") + tf_model.save_weights(tf_checkpoint_path) + pt_model = transformers.load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path) + pt_model = pt_model.to(torch_device) + + check_pt_tf_models(tf_model, pt_model, pt_inputs_dict) + def test_save_load(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -400,11 +544,8 @@ def default_feature_extractor(self): @slow def test_inference_for_pretraining(self): - # make random mask reproducible - # note that the same seed on CPU and on GPU doesn’t mean they spew the same random number sequences, - # as they both have fairly different PRNGs (for efficiency reasons). - # source: https://discuss.pytorch.org/t/random-seed-that-spans-across-devices/19735 - torch.manual_seed(2) + # make random mask reproducible across the PT and TF model + np.random.seed(2) model = ViTMAEForPreTraining.from_pretrained("facebook/vit-mae-base").to(torch_device) @@ -412,22 +553,22 @@ def test_inference_for_pretraining(self): image = prepare_img() inputs = feature_extractor(images=image, return_tensors="pt").to(torch_device) + # prepare a noise vector that will be also used for testing the TF model + # (this way we can ensure that the PT and TF models operate on the same inputs) + vit_mae_config = ViTMAEConfig() + num_patches = int((vit_mae_config.image_size // vit_mae_config.patch_size) ** 2) + noise = np.random.uniform(size=(1, num_patches)) + # forward pass with torch.no_grad(): - outputs = model(**inputs) + outputs = model(**inputs, noise=torch.from_numpy(noise)) # verify the logits expected_shape = torch.Size((1, 196, 768)) self.assertEqual(outputs.logits.shape, expected_shape) - expected_slice_cpu = torch.tensor( - [[0.7366, -1.3663, -0.2844], [0.7919, -1.3839, -0.3241], [0.4313, -0.7168, -0.2878]] + expected_slice = torch.tensor( + [[-0.0548, -1.7023, -0.9325], [0.3721, -0.5670, -0.2233], [0.8235, -1.3878, -0.3524]] ) - expected_slice_gpu = torch.tensor( - [[0.8948, -1.0680, 0.0030], [0.9758, -1.1181, -0.0290], [1.0602, -1.1522, -0.0528]] - ) - - # set expected slice depending on device - expected_slice = expected_slice_cpu if torch_device == "cpu" else expected_slice_gpu self.assertTrue(torch.allclose(outputs.logits[0, :3, :3], expected_slice.to(torch_device), atol=1e-4)) From 781af7362bac462f2174dfc82efdca5128ecc2f2 Mon Sep 17 00:00:00 2001 From: akashe Date: Tue, 29 Mar 2022 22:54:25 +0530 Subject: [PATCH 17/54] added typehints for RAG pytorch models (#16416) --- src/transformers/models/rag/modeling_rag.py | 132 ++++++++++---------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py index d3c08f616525..e3a303e1fd2f 100644 --- a/src/transformers/models/rag/modeling_rag.py +++ b/src/transformers/models/rag/modeling_rag.py @@ -767,25 +767,25 @@ def set_context_encoder_for_training(self, ctx_encoder: PreTrainedModel): @replace_return_docstrings(output_type=RetrievAugLMMarginOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - encoder_outputs=None, - decoder_input_ids=None, - decoder_attention_mask=None, - past_key_values=None, - context_input_ids=None, - context_attention_mask=None, - doc_scores=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - output_retrieved=None, - exclude_bos_score=None, - reduce_loss=None, - labels=None, - n_docs=None, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.Tensor]]] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.BoolTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + context_input_ids: Optional[torch.LongTensor] = None, + context_attention_mask: Optional[torch.LongTensor] = None, + doc_scores: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_retrieved: Optional[bool] = None, + exclude_bos_score: Optional[bool] = None, + reduce_loss: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + n_docs: Optional[int] = None, **kwargs # needs kwargs for generation - ): + ) -> RetrievAugLMMarginOutput: r""" exclude_bos_score (`bool`, *optional*): Only relevant if `labels` is passed. If `True`, the score of the BOS token is disregarded when computing @@ -910,15 +910,15 @@ def generate( self, input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, - context_input_ids=None, - context_attention_mask=None, - doc_scores=None, - do_deduplication=None, # defaults to True - num_return_sequences=None, # defaults to 1 - num_beams=None, # defaults to 1 - n_docs=None, + context_input_ids: Optional[torch.LongTensor] = None, + context_attention_mask: Optional[torch.LongTensor] = None, + doc_scores: Optional[torch.FloatTensor] = None, + do_deduplication: Optional[bool] = None, # defaults to True + num_return_sequences: Optional[int] = None, # defaults to 1 + num_beams: Optional[int] = None, # defaults to 1 + n_docs: Optional[int] = None, **model_kwargs - ): + ) -> torch.LongTensor: """ Implements RAG sequence "thorough" decoding. Read the [`~generation_utils.GenerationMixin.generate`]` documentation for more information on how to set other generate input parameters. @@ -1234,25 +1234,25 @@ def marginalize(self, seq_logits, doc_scores, n_docs=None): @replace_return_docstrings(output_type=RetrievAugLMMarginOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - encoder_outputs=None, - decoder_input_ids=None, - decoder_attention_mask=None, - past_key_values=None, - context_input_ids=None, - context_attention_mask=None, - doc_scores=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - output_retrieved=None, - do_marginalize=None, - reduce_loss=None, - labels=None, - n_docs=None, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.Tensor]]] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.BoolTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + context_input_ids: Optional[torch.LongTensor] = None, + context_attention_mask: Optional[torch.LongTensor] = None, + doc_scores: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_retrieved: Optional[bool] = None, + do_marginalize: Optional[bool] = None, + reduce_loss: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + n_docs: Optional[int] = None, **kwargs # needs kwargs for generation - ): + ) -> RetrievAugLMMarginOutput: r""" do_marginalize (`bool`, *optional*): If `True`, the logits are marginalized over all documents by making use of @@ -1377,27 +1377,27 @@ def generate( self, input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, - context_input_ids=None, - context_attention_mask=None, - doc_scores=None, - max_length=None, - min_length=None, - early_stopping=None, - use_cache=None, - num_beams=None, - num_beam_groups=None, - diversity_penalty=None, - bos_token_id=None, - pad_token_id=None, - eos_token_id=None, - length_penalty=None, - no_repeat_ngram_size=None, - encoder_no_repeat_ngram_size=None, - repetition_penalty=None, - bad_words_ids=None, - num_return_sequences=None, - decoder_start_token_id=None, - n_docs=None, + context_input_ids: Optional[torch.LongTensor] = None, + context_attention_mask: Optional[torch.LongTensor] = None, + doc_scores: Optional[torch.FloatTensor] = None, + max_length: Optional[int] = None, + min_length: Optional[int] = None, + early_stopping: Optional[bool] = None, + use_cache: Optional[bool] = None, + num_beams: Optional[int] = None, + num_beam_groups: Optional[int] = None, + diversity_penalty: Optional[float] = None, + bos_token_id: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + length_penalty: Optional[float] = None, + no_repeat_ngram_size: Optional[int] = None, + encoder_no_repeat_ngram_size: Optional[int] = None, + repetition_penalty: Optional[float] = None, + bad_words_ids: Optional[List[List[int]]] = None, + num_return_sequences: Optional[int] = None, + decoder_start_token_id: Optional[int] = None, + n_docs: Optional[int] = None, prefix_allowed_tokens_fn: Callable[[int, torch.Tensor], List[int]] = None, logits_processor: Optional[LogitsProcessorList] = LogitsProcessorList(), stopping_criteria: Optional[StoppingCriteriaList] = StoppingCriteriaList(), @@ -1406,7 +1406,7 @@ def generate( remove_invalid_values: Optional[bool] = None, exponential_decay_length_penalty: Optional[Tuple[Union[int, float]]] = None, **model_kwargs - ): + ) -> torch.LongTensor: """ Implements RAG token decoding. From d7c8ce57d43499f0042e63d5be2592c641354bad Mon Sep 17 00:00:00 2001 From: Sander Land <48946947+sanderland@users.noreply.github.com> Date: Tue, 29 Mar 2022 21:00:18 +0200 Subject: [PATCH 18/54] Avoid accessing .dataset of a DataLoader in Trainer (#16451) * Avoid accessing .dataset of a dataloader * style * fix * cleaning up, reverting some misunderstandings * black * add train_dataset argument to get_train_dataloader, and fix other instances of length checks * flake8 * address comments * fix bug * cleanup * add test * Update tests/trainer/test_trainer.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * under torch * merge * stylistic suggestion Co-authored-by: Sander Land Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/trainer.py | 65 ++++++++++++++++------------ src/transformers/trainer_callback.py | 2 +- src/transformers/utils/notebook.py | 5 +-- tests/trainer/test_trainer.py | 29 +++++++++++++ 4 files changed, 69 insertions(+), 32 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 1bf6fde9fc5e..993e5248635e 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -585,7 +585,7 @@ def _remove_unused_columns(self, dataset: "datasets.Dataset", description: Optio return dataset.remove_columns(ignored_columns) def _get_train_sampler(self) -> Optional[torch.utils.data.Sampler]: - if not has_length(self.train_dataset): + if self.train_dataset is None or not has_length(self.train_dataset): return None generator = None @@ -661,8 +661,8 @@ def get_train_dataloader(self) -> DataLoader: """ Returns the training [`~torch.utils.data.DataLoader`]. - Will use no sampler if `self.train_dataset` does not implement `__len__`, a random sampler (adapted to - distributed training if necessary) otherwise. + Will use no sampler if `train_dataset` does not implement `__len__`, a random sampler (adapted to distributed + training if necessary) otherwise. Subclass and override this method if you want to inject some custom behavior. """ @@ -937,11 +937,13 @@ def create_scheduler(self, num_training_steps: int, optimizer: torch.optim.Optim def num_examples(self, dataloader: DataLoader) -> int: """ - Helper to get number of samples in a [`~torch.utils.data.DataLoader`] by accessing its dataset. - - Will raise an exception if the underlying dataset does not implement method `__len__` + Helper to get number of samples in a [`~torch.utils.data.DataLoader`] by accessing its dataset. When + dataloader.dataset does not exist or has no length, estimates as best it can """ - return len(dataloader.dataset) + try: + return len(dataloader.dataset) + except (NameError, AttributeError, TypeError): # no dataset or length, estimate by length of dataloader + return len(dataloader) * self.args.per_device_train_batch_size def _hp_search_setup(self, trial: Union["optuna.Trial", Dict[str, Any]]): """HP search setup code""" @@ -1198,9 +1200,6 @@ def train( self._move_model_to_device(self.model, args.device) self.model_wrapped = self.model - # Keeping track whether we can can len() on the dataset or not - train_dataset_is_sized = has_length(self.train_dataset) - # Data loader and number of training steps train_dataloader = self.get_train_dataloader() @@ -1209,28 +1208,36 @@ def train( # number of training steps per epoch: num_update_steps_per_epoch # total number of training steps to execute: max_steps total_train_batch_size = args.train_batch_size * args.gradient_accumulation_steps * args.world_size - if train_dataset_is_sized: - num_update_steps_per_epoch = len(train_dataloader) // args.gradient_accumulation_steps + + len_dataloader = None + if has_length(train_dataloader): + len_dataloader = len(train_dataloader) + num_update_steps_per_epoch = len_dataloader // args.gradient_accumulation_steps num_update_steps_per_epoch = max(num_update_steps_per_epoch, 1) + num_examples = self.num_examples(train_dataloader) if args.max_steps > 0: max_steps = args.max_steps num_train_epochs = args.max_steps // num_update_steps_per_epoch + int( args.max_steps % num_update_steps_per_epoch > 0 ) - # May be slightly incorrect if the last batch in the training datalaoder has a smaller size but it's + # May be slightly incorrect if the last batch in the training dataloader has a smaller size but it's # the best we can do. num_train_samples = args.max_steps * total_train_batch_size else: max_steps = math.ceil(args.num_train_epochs * num_update_steps_per_epoch) num_train_epochs = math.ceil(args.num_train_epochs) - num_train_samples = len(self.train_dataset) * args.num_train_epochs - else: - # see __init__. max_steps is set when the dataset has no __len__ + num_train_samples = self.num_examples(train_dataloader) * args.num_train_epochs + elif args.max_steps > 0: # Rely on max_steps when dataloader does not have a working size max_steps = args.max_steps # Setting a very large number of epochs so we go as many times as necessary over the iterator. num_train_epochs = sys.maxsize num_update_steps_per_epoch = max_steps + num_examples = total_train_batch_size * args.max_steps num_train_samples = args.max_steps * total_train_batch_size + else: + raise ValueError( + f"args.max_steps must be set to a positive value if dataloader does not have a length, was {args.max_steps}" + ) if DebugOption.UNDERFLOW_OVERFLOW in self.args.debug: if self.args.n_gpu > 1: @@ -1281,10 +1288,6 @@ def train( # self.model_wrapped is DDP(Transformers Model), Deepspeed(Transformers Model), etc. # Train! - num_examples = ( - self.num_examples(train_dataloader) if train_dataset_is_sized else total_train_batch_size * args.max_steps - ) - logger.info("***** Running training *****") logger.info(f" Num examples = {num_examples}") logger.info(f" Num Epochs = {num_train_epochs}") @@ -1370,7 +1373,7 @@ def train( for epoch in range(epochs_trained, num_train_epochs): if isinstance(train_dataloader, DataLoader) and isinstance(train_dataloader.sampler, DistributedSampler): train_dataloader.sampler.set_epoch(epoch) - elif isinstance(train_dataloader.dataset, IterableDatasetShard): + elif hasattr(train_dataloader, "dataset") and isinstance(train_dataloader.dataset, IterableDatasetShard): train_dataloader.dataset.set_epoch(epoch) if is_torch_tpu_available(): @@ -1384,7 +1387,9 @@ def train( self._past = None steps_in_epoch = ( - len(epoch_iterator) if train_dataset_is_sized else args.max_steps * args.gradient_accumulation_steps + len(epoch_iterator) + if len_dataloader is not None + else args.max_steps * args.gradient_accumulation_steps ) self.control = self.callback_handler.on_epoch_begin(args, self.state, self.control) @@ -2407,10 +2412,10 @@ def evaluation_loop( elif args.bf16_full_eval: model = model.to(dtype=torch.bfloat16, device=args.device) - batch_size = dataloader.batch_size + batch_size = self.args.per_device_eval_batch_size logger.info(f"***** Running {description} *****") - if has_length(dataloader.dataset): + if has_length(dataloader): logger.info(f" Num examples = {self.num_examples(dataloader)}") else: logger.info(" Num examples: Unknown") @@ -2420,7 +2425,7 @@ def evaluation_loop( self.callback_handler.eval_dataloader = dataloader # Do this before wrapping. - eval_dataset = dataloader.dataset + eval_dataset = getattr(dataloader, "dataset", None) if is_torch_tpu_available(): dataloader = pl.ParallelLoader(dataloader, [args.device]).per_device_loader(args.device) @@ -2512,7 +2517,10 @@ def evaluation_loop( elif isinstance(eval_dataset, IterableDatasetShard) and hasattr(eval_dataset, "num_examples"): num_samples = eval_dataset.num_examples else: - num_samples = observed_num_examples + if has_length(dataloader): + num_samples = self.num_examples(dataloader) + else: # both len(dataloader.dataset) and len(dataloader) fail + num_samples = observed_num_examples # Number of losses has been rounded to a multiple of batch_size and in a distributed training, the number of # samplers has been rounded to a multiple of batch_size, so we truncate. @@ -2899,8 +2907,9 @@ def prediction_loop( """ args = self.args - if not has_length(dataloader.dataset): - raise ValueError("dataset must implement __len__") + if not has_length(dataloader): + raise ValueError("dataloader must implement a working __len__") + prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else args.prediction_loss_only # if eval is called w/o train init deepspeed here diff --git a/src/transformers/trainer_callback.py b/src/transformers/trainer_callback.py index ec344341bce6..92abe1ed5063 100644 --- a/src/transformers/trainer_callback.py +++ b/src/transformers/trainer_callback.py @@ -473,7 +473,7 @@ def on_step_end(self, args, state, control, **kwargs): self.current_step = state.global_step def on_prediction_step(self, args, state, control, eval_dataloader=None, **kwargs): - if state.is_local_process_zero and has_length(eval_dataloader.dataset): + if state.is_local_process_zero and has_length(eval_dataloader): if self.prediction_bar is None: self.prediction_bar = tqdm(total=len(eval_dataloader), leave=self.training_bar is None) self.prediction_bar.update(1) diff --git a/src/transformers/utils/notebook.py b/src/transformers/utils/notebook.py index 779446f5f104..0ffbdc8deecf 100644 --- a/src/transformers/utils/notebook.py +++ b/src/transformers/utils/notebook.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections import re import time from typing import Optional @@ -21,7 +20,7 @@ import IPython.display as disp from ..trainer_callback import TrainerCallback -from ..trainer_utils import IntervalStrategy +from ..trainer_utils import IntervalStrategy, has_length def format_time(t): @@ -294,7 +293,7 @@ def on_step_end(self, args, state, control, **kwargs): self._force_next_update = False def on_prediction_step(self, args, state, control, eval_dataloader=None, **kwargs): - if not isinstance(eval_dataloader.dataset, collections.abc.Sized): + if not has_length(eval_dataloader): return if self.prediction_bar is None: if self.training_tracker is not None: diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index ec044ea1daa7..afe97701d29b 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -189,6 +189,26 @@ def __iter__(self): yield self.dataset[self.current_sample] self.current_sample += 1 + class MultiLoader: + def __init__(self, loaders): + self.loaders = loaders + + def __len__(self): + return sum(len(loader) for loader in self.loaders) + + def __iter__(self): + for loader in self.loaders: + yield from loader + + class CustomDataloaderTrainer(Trainer): + def get_train_dataloader(self): + dataloaders = [super().get_train_dataloader(), super().get_train_dataloader()] + return MultiLoader(dataloaders) + + def get_eval_dataloader(self, eval_dataset): + dataloaders = [super().get_eval_dataloader(eval_dataset), super().get_eval_dataloader(eval_dataset)] + return MultiLoader(dataloaders) + class RegressionModel(nn.Module): def __init__(self, a=0, b=0, double_output=False): super().__init__() @@ -647,6 +667,15 @@ def test_train_and_eval_dataloaders(self): new_eval_dataset = RegressionDataset(length=128) self.assertEqual(len(trainer.get_eval_dataloader(new_eval_dataset)), 128 // (32 * n_gpu)) + # tests that we do not require dataloader to have a .dataset attribute + def test_dataloader_without_dataset(self): + train_dataset = RegressionDataset(length=128) + trainer = CustomDataloaderTrainer( + model=RegressionModel(), train_dataset=train_dataset, eval_dataset=train_dataset + ) + trainer.train() + trainer.evaluate() + def test_sampler_seed(self): # nb: we don't want to inherit from IterableDataset to hit the right code path class DummyDataset(torch.utils.data.Dataset): From ee18d4d2a994fe8c70773f0d7a65f71d1e5b1370 Mon Sep 17 00:00:00 2001 From: Christopher Akiki Date: Tue, 29 Mar 2022 21:35:25 +0200 Subject: [PATCH 19/54] TF GPT2: clearer model variable naming with @unpack_inputs (#16311) * add unpack_inputs decorator to Main Layer * add unpack_inputs decorator to Model * add unpack_inputs decorator to LMHead Model * add unpack_inputs decorator to Double Head Model * add unpack_inputs decorator to Sequence Classification Model * run fixup recipe * make unpack_inputs the first decorator --- .../models/gpt2/modeling_tf_gpt2.py | 284 ++++++------------ 1 file changed, 90 insertions(+), 194 deletions(-) diff --git a/src/transformers/models/gpt2/modeling_tf_gpt2.py b/src/transformers/models/gpt2/modeling_tf_gpt2.py index 4d3734e78049..88b4fb5ed607 100644 --- a/src/transformers/models/gpt2/modeling_tf_gpt2.py +++ b/src/transformers/models/gpt2/modeling_tf_gpt2.py @@ -37,8 +37,8 @@ TFSequenceSummary, TFSharedEmbeddings, get_initializer, - input_processing, keras_serializable, + unpack_inputs, ) from ...tf_utils import shape_list from ...utils import ( @@ -350,6 +350,7 @@ def _prune_heads(self, heads_to_prune): """ raise NotImplementedError + @unpack_inputs def call( self, input_ids: Optional[TFModelInputType] = None, @@ -368,55 +369,34 @@ def call( training: Optional[bool] = False, **kwargs, ) -> Union[TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor]]: - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - past=past, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - inputs["input_ids"] = tf.reshape(inputs["input_ids"], [-1, input_shape[-1]]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") - if inputs["past"] is None: + if past is None: past_length = 0 - inputs["past"] = [None] * len(self.h) + past = [None] * len(self.h) else: - past_length = shape_list(inputs["past"][0][0])[-2] + past_length = shape_list(past[0][0])[-2] - if inputs["position_ids"] is None: - inputs["position_ids"] = tf.expand_dims(tf.range(past_length, input_shape[-1] + past_length), axis=0) + if position_ids is None: + position_ids = tf.expand_dims(tf.range(past_length, input_shape[-1] + past_length), axis=0) - if inputs["attention_mask"] is not None: + if attention_mask is not None: # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] # this attention mask is more simple than the triangular masking of causal attention # used in OpenAI GPT, we just need to prepare the broadcast dimension here. - attention_mask_shape = shape_list(inputs["attention_mask"]) - inputs["attention_mask"] = tf.reshape( - inputs["attention_mask"], (attention_mask_shape[0], 1, 1, attention_mask_shape[1]) - ) + attention_mask_shape = shape_list(attention_mask) + attention_mask = tf.reshape(attention_mask, (attention_mask_shape[0], 1, 1, attention_mask_shape[1])) # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for @@ -424,24 +404,20 @@ def call( # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. one_cst = tf.constant(1.0) - inputs["attention_mask"] = tf.cast(inputs["attention_mask"], dtype=one_cst.dtype) - inputs["attention_mask"] = tf.multiply( - tf.subtract(one_cst, inputs["attention_mask"]), tf.constant(-10000.0) - ) + attention_mask = tf.cast(attention_mask, dtype=one_cst.dtype) + attention_mask = tf.multiply(tf.subtract(one_cst, attention_mask), tf.constant(-10000.0)) # Copied from `modeling_tf_t5.py` with -1e9 -> -10000 - if self.config.add_cross_attention and inputs["encoder_attention_mask"] is not None: + if self.config.add_cross_attention and encoder_attention_mask is not None: # If a 2D ou 3D attention mask is provided for the cross-attention # we need to make broadcastable to [batch_size, num_heads, mask_seq_length, mask_seq_length] # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] - inputs["encoder_attention_mask"] = tf.cast( - inputs["encoder_attention_mask"], dtype=inputs["encoder_hidden_states"].dtype - ) - num_dims_encoder_attention_mask = len(shape_list(inputs["encoder_attention_mask"])) + encoder_attention_mask = tf.cast(encoder_attention_mask, dtype=encoder_hidden_states.dtype) + num_dims_encoder_attention_mask = len(shape_list(encoder_attention_mask)) if num_dims_encoder_attention_mask == 3: - encoder_extended_attention_mask = inputs["encoder_attention_mask"][:, None, :, :] + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] if num_dims_encoder_attention_mask == 2: - encoder_extended_attention_mask = inputs["encoder_attention_mask"][:, None, None, :] + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 @@ -452,66 +428,64 @@ def call( else: encoder_extended_attention_mask = None - inputs["encoder_attention_mask"] = encoder_extended_attention_mask + encoder_attention_mask = encoder_extended_attention_mask # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head # attention_probs has shape bsz x n_heads x N x N # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] - if inputs["head_mask"] is not None: + if head_mask is not None: raise NotImplementedError else: - inputs["head_mask"] = [None] * self.num_hidden_layers + head_mask = [None] * self.num_hidden_layers # head_mask = tf.constant([0] * self.num_hidden_layers) - inputs["position_ids"] = tf.reshape(inputs["position_ids"], [-1, shape_list(inputs["position_ids"])[-1]]) + position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]]) - if inputs["inputs_embeds"] is None: - inputs["inputs_embeds"] = self.wte(inputs["input_ids"], mode="embedding") + if inputs_embeds is None: + inputs_embeds = self.wte(input_ids, mode="embedding") - position_embeds = tf.gather(self.wpe, inputs["position_ids"]) + position_embeds = tf.gather(self.wpe, position_ids) - if inputs["token_type_ids"] is not None: - inputs["token_type_ids"] = tf.reshape( - inputs["token_type_ids"], [-1, shape_list(inputs["token_type_ids"])[-1]] - ) - token_type_embeds = self.wte(inputs["token_type_ids"], mode="embedding") + if token_type_ids is not None: + token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]]) + token_type_embeds = self.wte(token_type_ids, mode="embedding") else: token_type_embeds = tf.constant(0.0) - position_embeds = tf.cast(position_embeds, dtype=inputs["inputs_embeds"].dtype) - token_type_embeds = tf.cast(token_type_embeds, dtype=inputs["inputs_embeds"].dtype) - hidden_states = inputs["inputs_embeds"] + position_embeds + token_type_embeds - hidden_states = self.drop(hidden_states, training=inputs["training"]) + position_embeds = tf.cast(position_embeds, dtype=inputs_embeds.dtype) + token_type_embeds = tf.cast(token_type_embeds, dtype=inputs_embeds.dtype) + hidden_states = inputs_embeds + position_embeds + token_type_embeds + hidden_states = self.drop(hidden_states, training=training) output_shape = input_shape + [shape_list(hidden_states)[-1]] - presents = () if inputs["use_cache"] else None - all_attentions = () if inputs["output_attentions"] else None - all_cross_attentions = () if inputs["output_attentions"] and self.config.add_cross_attention else None - all_hidden_states = () if inputs["output_hidden_states"] else None - for i, (block, layer_past) in enumerate(zip(self.h, inputs["past"])): - if inputs["output_hidden_states"]: + presents = () if use_cache else None + all_attentions = () if output_attentions else None + all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None + all_hidden_states = () if output_hidden_states else None + for i, (block, layer_past) in enumerate(zip(self.h, past)): + if output_hidden_states: all_hidden_states = all_hidden_states + (tf.reshape(hidden_states, output_shape),) outputs = block( hidden_states, layer_past, - inputs["attention_mask"], - inputs["head_mask"][i], - inputs["encoder_hidden_states"], - inputs["encoder_attention_mask"], - inputs["use_cache"], - inputs["output_attentions"], - training=inputs["training"], + attention_mask, + head_mask[i], + encoder_hidden_states, + encoder_attention_mask, + use_cache, + output_attentions, + training=training, ) hidden_states, present = outputs[:2] - if inputs["use_cache"]: + if use_cache: presents = presents + (present,) - if inputs["output_attentions"]: + if output_attentions: all_attentions = all_attentions + (outputs[2],) if self.config.add_cross_attention and encoder_hidden_states is not None: all_cross_attentions = all_cross_attentions + (outputs[3],) @@ -520,15 +494,15 @@ def call( hidden_states = tf.reshape(hidden_states, output_shape) # Add last hidden state - if inputs["output_hidden_states"]: + if output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) - if inputs["output_attentions"]: + if output_attentions: # let the number of heads free (-1) so we can extract attention even after head pruning attention_output_shape = input_shape[:-1] + [-1] + shape_list(all_attentions[0])[-2:] all_attentions = tuple(tf.reshape(t, attention_output_shape) for t in all_attentions) - if not inputs["return_dict"]: + if not return_dict: return tuple( v for v in [hidden_states, presents, all_hidden_states, all_attentions, all_cross_attentions] @@ -732,6 +706,7 @@ def __init__(self, config, *inputs, **kwargs): super().__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name="transformer") + @unpack_inputs @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -777,9 +752,8 @@ def call( If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past`). Set to `False` during training, `True` during generation """ - inputs = input_processing( - func=self.call, - config=self.config, + + outputs = self.transformer( input_ids=input_ids, past=past, attention_mask=attention_mask, @@ -794,23 +768,6 @@ def call( output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - outputs = self.transformer( - input_ids=inputs["input_ids"], - past=inputs["past"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - encoder_hidden_states=inputs["encoder_hidden_states"], - encoder_attention_mask=inputs["encoder_attention_mask"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) return outputs @@ -938,6 +895,7 @@ def _update_model_kwargs_for_xla_generation(self, outputs, model_kwargs, current return model_kwargs + @unpack_inputs @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -987,9 +945,8 @@ def call( Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. """ - inputs = input_processing( - func=self.call, - config=self.config, + + transformer_outputs = self.transformer( input_ids=input_ids, past=past, attention_mask=attention_mask, @@ -1003,37 +960,19 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - transformer_outputs = self.transformer( - input_ids=inputs["input_ids"], - past=inputs["past"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - encoder_hidden_states=inputs["encoder_hidden_states"], - encoder_attention_mask=inputs["encoder_attention_mask"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) hidden_states = transformer_outputs[0] logits = self.transformer.wte(hidden_states, mode="linear") loss = None - if inputs["labels"] is not None: + if labels is not None: # shift labels to the left and cut last logit token shifted_logits = logits[:, :-1] - labels = inputs["labels"][:, 1:] + labels = labels[:, 1:] loss = self.hf_compute_loss(labels, shifted_logits) - if not inputs["return_dict"]: + if not return_dict: output = (logits,) + transformer_outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1081,6 +1020,7 @@ def __init__(self, config, *inputs, **kwargs): config, initializer_range=config.initializer_range, name="multiple_choice_head" ) + @unpack_inputs @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=TFGPT2DoubleHeadsModelOutput, config_class=_CONFIG_FOR_DOC) def call( @@ -1133,64 +1073,40 @@ def call( >>> outputs = model(input_ids, mc_token_ids=mc_token_ids) >>> lm_prediction_scores, mc_prediction_scores = outputs[:2] ```""" - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - past=past, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - mc_token_ids=mc_token_ids, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None: - input_shapes = shape_list(inputs["input_ids"]) + if input_ids is not None: + input_shapes = shape_list(input_ids) else: - input_shapes = shape_list(inputs["inputs_embeds"])[:-1] + input_shapes = shape_list(inputs_embeds)[:-1] seq_length = input_shapes[-1] - flat_input_ids = tf.reshape(inputs["input_ids"], (-1, seq_length)) if inputs["input_ids"] is not None else None - flat_attention_mask = ( - tf.reshape(inputs["attention_mask"], (-1, seq_length)) if inputs["attention_mask"] is not None else None - ) - flat_token_type_ids = ( - tf.reshape(inputs["token_type_ids"], (-1, seq_length)) if inputs["token_type_ids"] is not None else None - ) - flat_position_ids = ( - tf.reshape(inputs["position_ids"], (-1, seq_length)) if inputs["position_ids"] is not None else None - ) + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None + flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None + flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None + flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None transformer_outputs = self.transformer( input_ids=flat_input_ids, - past=inputs["past"], + past=past, attention_mask=flat_attention_mask, token_type_ids=flat_token_type_ids, position_ids=flat_position_ids, - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], + head_mask=head_mask, + inputs_embeds=inputs_embeds, encoder_hidden_states=None, encoder_attention_mask=None, - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, ) hidden_states = transformer_outputs[0] hidden_states = tf.reshape(hidden_states, input_shapes + shape_list(hidden_states)[-1:]) lm_logits = self.transformer.wte(hidden_states, mode="linear") - mc_logits = self.multiple_choice_head(hidden_states, inputs["mc_token_ids"], training=inputs["training"]) + mc_logits = self.multiple_choice_head(hidden_states, mc_token_ids, training=training) mc_logits = tf.squeeze(mc_logits, axis=-1) - if not inputs["return_dict"]: + if not return_dict: return (lm_logits, mc_logits) + transformer_outputs[1:] return TFGPT2DoubleHeadsModelOutput( @@ -1256,6 +1172,7 @@ def __init__(self, config, *inputs, **kwargs): ) self.transformer = TFGPT2MainLayer(config, name="transformer") + @unpack_inputs @add_start_docstrings_to_model_forward(GPT2_INPUTS_DOCSTRING) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1285,9 +1202,7 @@ def call( Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. """ - inputs = input_processing( - func=self.call, - config=self.config, + transformer_outputs = self.transformer( input_ids=input_ids, past=past, attention_mask=attention_mask, @@ -1299,24 +1214,7 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - - transformer_outputs = self.transformer( - input_ids=inputs["input_ids"], - past=inputs["past"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) hidden_states = transformer_outputs[0] @@ -1326,12 +1224,12 @@ def call( if self.config.pad_token_id is None: sequence_lengths = -1 else: - if inputs["input_ids"] is not None: + if input_ids is not None: sequence_lengths = ( tf.reduce_sum( tf.cast( - tf.math.not_equal(inputs["input_ids"], self.config.pad_token_id), - dtype=inputs["input_ids"].dtype, + tf.math.not_equal(input_ids, self.config.pad_token_id), + dtype=input_ids.dtype, ), -1, keepdims=False, @@ -1347,7 +1245,7 @@ def call( ) loss = None - if inputs["labels"] is not None: + if labels is not None: assert ( self.config.pad_token_id is not None or logits_shape[0] == 1 ), "Cannot handle batch sizes > 1 if no padding token is defined." @@ -1355,12 +1253,10 @@ def call( if not tf.is_tensor(sequence_lengths): in_logits = logits[0 : logits_shape[0], sequence_lengths] - loss = self.hf_compute_loss( - tf.reshape(inputs["labels"], [-1]), tf.reshape(in_logits, [-1, self.num_labels]) - ) + loss = self.hf_compute_loss(tf.reshape(labels, [-1]), tf.reshape(in_logits, [-1, self.num_labels])) pooled_logits = in_logits if in_logits is not None else logits - if not inputs["return_dict"]: + if not return_dict: output = (pooled_logits,) + transformer_outputs[1:] return ((loss,) + output) if loss is not None else output From 2b483230a1f79e2127b62032ef42d388ba1f991f Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 29 Mar 2022 22:12:27 +0200 Subject: [PATCH 20/54] Raise diff tolerance value for TFViTMAEModelTest (#16483) * Raise diff tolerance value Co-authored-by: ydshieh --- tests/vit_mae/test_modeling_tf_vit_mae.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/vit_mae/test_modeling_tf_vit_mae.py b/tests/vit_mae/test_modeling_tf_vit_mae.py index c0fb84856934..e978fabb334f 100644 --- a/tests/vit_mae/test_modeling_tf_vit_mae.py +++ b/tests/vit_mae/test_modeling_tf_vit_mae.py @@ -427,7 +427,9 @@ def check_outputs(tf_outputs, pt_outputs, model_class, names): tf_outputs[pt_nans] = 0 max_diff = np.amax(np.abs(tf_outputs - pt_outputs)) - self.assertLessEqual(max_diff, 1e-5) + # Set a higher tolerance (2e-5) here than the one in the common test (1e-5). + # TODO: A deeper look to decide the best (common) tolerance for the test to be strict but not too flaky. + self.assertLessEqual(max_diff, 2e-5) else: raise ValueError( f"`tf_outputs` should be a `tuple` or an instance of `tf.Tensor`. Got {type(tf_outputs)} instead." From 277d49a590b6745ec82460eea3f33a825a89051c Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Tue, 29 Mar 2022 16:07:31 -0700 Subject: [PATCH 21/54] Do not initialize `torch.distributed` process group if one is already initailized (#16487) * Do not initialize torch process group twice * Apply suggestions from code review --- src/transformers/sagemaker/training_args_sm.py | 8 +++++++- src/transformers/training_args.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/transformers/sagemaker/training_args_sm.py b/src/transformers/sagemaker/training_args_sm.py index 992f3d4fce30..f6c57d8f8577 100644 --- a/src/transformers/sagemaker/training_args_sm.py +++ b/src/transformers/sagemaker/training_args_sm.py @@ -77,6 +77,11 @@ def __post_init__(self): @cached_property def _setup_devices(self) -> "torch.device": logger.info("PyTorch: setting up devices") + if torch.distributed.is_initialized() and self.local_rank == -1: + logger.warning( + "torch.distributed process group is initialized, but local_rank == -1. " + "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch" + ) if self.no_cuda: device = torch.device("cpu") self._n_gpu = 0 @@ -105,7 +110,8 @@ def _setup_devices(self) -> "torch.device": else: # Here, we'll use torch.distributed. # Initializes the distributed backend which will take care of synchronizing nodes/GPUs - torch.distributed.init_process_group(backend="nccl") + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group(backend="nccl") device = torch.device("cuda", self.local_rank) self._n_gpu = 1 diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index 2087fbb7ace0..b0e6fbc6e85e 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -1022,10 +1022,15 @@ def eval_batch_size(self) -> int: @torch_required def _setup_devices(self) -> "torch.device": logger.info("PyTorch: setting up devices") + if torch.distributed.is_initialized() and self.local_rank == -1: + logger.warning( + "torch.distributed process group is initialized, but local_rank == -1. " + "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch" + ) if self.no_cuda: device = torch.device("cpu") self._n_gpu = 0 - if self.local_rank != -1: + if self.local_rank != -1 and not torch.distributed.is_initialized(): # Initializes distributed backend for cpu if self.xpu_backend not in ("mpi", "ccl"): raise ValueError( @@ -1076,7 +1081,8 @@ def _setup_devices(self) -> "torch.device": else: # Here, we'll use torch.distributed. # Initializes the distributed backend which will take care of synchronizing nodes/GPUs - torch.distributed.init_process_group(backend="nccl") + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group(backend="nccl") device = torch.device("cuda", self.local_rank) self._n_gpu = 1 From ffd19ee1de36188c6208855160b5ff930caa00c0 Mon Sep 17 00:00:00 2001 From: Dahlbomii <101373053+Dahlbomii@users.noreply.github.com> Date: Wed, 30 Mar 2022 06:03:54 -0700 Subject: [PATCH 22/54] TF GPT-J Type hints and TF decorator (#16488) * Type hints and TF decorator added * Type hints and TF decorator added * make style Co-authored-by: matt --- .../models/gptj/modeling_tf_gptj.py | 222 +++++++----------- 1 file changed, 80 insertions(+), 142 deletions(-) diff --git a/src/transformers/models/gptj/modeling_tf_gptj.py b/src/transformers/models/gptj/modeling_tf_gptj.py index 4bb9ec4a81ed..e6d9a4f308ac 100644 --- a/src/transformers/models/gptj/modeling_tf_gptj.py +++ b/src/transformers/models/gptj/modeling_tf_gptj.py @@ -14,8 +14,9 @@ # limitations under the License. """ TF 2.0 GPT-J model.""" -from typing import Optional, Tuple +from typing import Optional, Tuple, Union +import numpy as np import tensorflow as tf from ...activations_tf import get_tf_activation @@ -33,6 +34,7 @@ ) from ...modeling_tf_utils import ( TFCausalLanguageModelingLoss, + TFModelInputType, TFPreTrainedModel, TFQuestionAnsweringLoss, TFSequenceClassificationLoss, @@ -40,6 +42,7 @@ get_initializer, input_processing, keras_serializable, + unpack_inputs, ) from ...tf_utils import shape_list from ...utils import logging @@ -670,6 +673,7 @@ def __init__(self, config, *inputs, **kwargs): super().__init__(config, *inputs, **kwargs) self.transformer = TFGPTJMainLayer(config, name="transformer") + @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -679,18 +683,18 @@ def __init__(self, config, *inputs, **kwargs): ) def call( self, - input_ids=None, - past_key_values=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, **kwargs, ): r""" @@ -698,9 +702,8 @@ def call( If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past`). Set to `False` during training, `True` during generation """ - inputs = input_processing( - func=self.call, - config=self.config, + + outputs = self.transformer( input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, @@ -713,21 +716,6 @@ def call( output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - outputs = self.transformer( - input_ids=inputs["input_ids"], - past_key_values=inputs["past_key_values"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) return outputs @@ -793,6 +781,7 @@ def prepare_inputs_for_generation(self, inputs, past=None, use_cache=None, use_x "use_cache": use_cache, } + @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -802,19 +791,19 @@ def prepare_inputs_for_generation(self, inputs, past=None, use_cache=None, use_x ) def call( self, - input_ids=None, - past_key_values=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - labels=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + labels: Optional[Union[np.ndarray, tf.Tensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, **kwargs, ): r""" @@ -823,9 +812,8 @@ def call( `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` """ - inputs = input_processing( - func=self.call, - config=self.config, + + transformer_outputs = self.transformer( input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, @@ -833,39 +821,23 @@ def call( position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, - labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - transformer_outputs = self.transformer( - input_ids=inputs["input_ids"], - past_key_values=inputs["past_key_values"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) hidden_states = transformer_outputs[0] lm_logits = self.lm_head(hidden_states) loss = None - if inputs["labels"] is not None: + if labels is not None: # shift labels to the left and cut last logit token shifted_logits = lm_logits[:, :-1] - labels = inputs["labels"][:, 1:] + labels = labels[:, 1:] loss = self.hf_compute_loss(labels, shifted_logits) - if not inputs["return_dict"]: + if not return_dict: output = (lm_logits,) + transformer_outputs[1:] return ((loss,) + output) if loss is not None else output @@ -914,6 +886,7 @@ def __init__(self, config, *inputs, **kwargs): name="score", ) + @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -923,19 +896,19 @@ def __init__(self, config, *inputs, **kwargs): ) def call( self, - input_ids=None, - past_key_values=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - labels=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + labels: Optional[Union[np.ndarray, tf.Tensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, **kwargs, ): r""" @@ -944,9 +917,8 @@ def call( config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). """ - inputs = input_processing( - func=self.call, - config=self.config, + + transformer_outputs = self.transformer( input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, @@ -954,27 +926,11 @@ def call( position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, - labels=labels, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - transformer_outputs = self.transformer( - input_ids=inputs["input_ids"], - past_key_values=inputs["past_key_values"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) hidden_states = transformer_outputs[0] logits = self.score(hidden_states) @@ -983,12 +939,12 @@ def call( if self.config.pad_token_id is None: sequence_lengths = -1 else: - if inputs["input_ids"] is not None: + if input_ids is not None: sequence_lengths = ( tf.reduce_sum( tf.cast( - tf.math.not_equal(inputs["input_ids"], self.config.pad_token_id), - dtype=inputs["input_ids"].dtype, + tf.math.not_equal(input_ids, self.config.pad_token_id), + dtype=input_ids.dtype, ), -1, keepdims=False, @@ -1004,7 +960,7 @@ def call( ) loss = None - if inputs["labels"] is not None: + if labels is not None: assert ( self.config.pad_token_id is not None or logits_shape[0] == 1 ), "Cannot handle batch sizes > 1 if no padding token is defined." @@ -1012,12 +968,10 @@ def call( if not tf.is_tensor(sequence_lengths): in_logits = logits[0 : logits_shape[0], sequence_lengths] - loss = self.hf_compute_loss( - tf.reshape(inputs["labels"], [-1]), tf.reshape(in_logits, [-1, self.num_labels]) - ) + loss = self.hf_compute_loss(tf.reshape(labels, [-1]), tf.reshape(in_logits, [-1, self.num_labels])) pooled_logits = in_logits if in_logits is not None else logits - if not inputs["return_dict"]: + if not return_dict: output = (pooled_logits,) + transformer_outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1057,6 +1011,7 @@ def __init__(self, config, *inputs, **kwargs): self.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="qa_outputs" ) + @unpack_inputs @add_start_docstrings_to_model_forward(GPTJ_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1066,19 +1021,19 @@ def __init__(self, config, *inputs, **kwargs): ) def call( self, - input_ids=None, - past_key_values=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - start_positions=None, - end_positions=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + start_positions: Optional[Union[np.ndarray, tf.Tensor]] = None, + end_positions: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, **kwargs, ): r""" @@ -1091,9 +1046,8 @@ def call( Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. """ - inputs = input_processing( - func=self.call, - config=self.config, + + transformer_outputs = self.transformer( input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, @@ -1101,26 +1055,10 @@ def call( position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, - start_positions=start_positions, - end_positions=end_positions, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - transformer_outputs = self.transformer( - input_ids=inputs["input_ids"], - past_key_values=inputs["past_key_values"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) sequence_output = transformer_outputs[0] @@ -1130,12 +1068,12 @@ def call( end_logits = tf.squeeze(end_logits, axis=-1) loss = None - if inputs["start_positions"] is not None and inputs["end_positions"] is not None: - labels = {"start_position": inputs["start_positions"]} - labels["end_position"] = inputs["end_positions"] + if start_positions is not None and end_positions is not None: + labels = {"start_position": start_positions} + labels["end_position"] = end_positions loss = self.hf_compute_loss(labels, (start_logits, end_logits)) - if not inputs["return_dict"]: + if not return_dict: output = (start_logits, end_logits) + transformer_outputs[2:] return ((loss,) + output) if loss is not None else output From 147c8166852db64de12b851b8307f44c9e8fe0dd Mon Sep 17 00:00:00 2001 From: Aditya Kane <64411306+AdityaKane2001@users.noreply.github.com> Date: Wed, 30 Mar 2022 19:36:32 +0530 Subject: [PATCH 23/54] Nit: MCSCOCO -> MS COCO (#16481) --- src/transformers/models/lxmert/modeling_lxmert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/lxmert/modeling_lxmert.py b/src/transformers/models/lxmert/modeling_lxmert.py index 54db04c51d5f..c9b2541251e8 100644 --- a/src/transformers/models/lxmert/modeling_lxmert.py +++ b/src/transformers/models/lxmert/modeling_lxmert.py @@ -803,7 +803,7 @@ def _init_weights(self, module): The LXMERT model was proposed in [LXMERT: Learning Cross-Modality Encoder Representations from Transformers](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal. It's a vision and language transformer - model, pretrained on a variety of multi-modal datasets comprising of GQA, VQAv2.0, MCSCOCO captions, and Visual + model, pretrained on a variety of multi-modal datasets comprising of GQA, VQAv2.0, MSCOCO captions, and Visual genome, using a combination of masked language modeling, region of interest feature regression, cross entropy loss for question answering attribute prediction, and object tag prediction. From d04adc3521c87c6de48793afee2830aa3e220f9f Mon Sep 17 00:00:00 2001 From: dctelus <93535080+dctelus@users.noreply.github.com> Date: Wed, 30 Mar 2022 11:41:04 -0400 Subject: [PATCH 24/54] Add length to PreTrainedTokenizer train_new_from_iterator (#16493) --- src/transformers/tokenization_utils_fast.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py index 1ee2a44d9b9f..4f85a842dd3d 100644 --- a/src/transformers/tokenization_utils_fast.py +++ b/src/transformers/tokenization_utils_fast.py @@ -601,7 +601,13 @@ def _save_pretrained( return file_names def train_new_from_iterator( - self, text_iterator, vocab_size, new_special_tokens=None, special_tokens_map=None, **kwargs + self, + text_iterator, + vocab_size, + length=None, + new_special_tokens=None, + special_tokens_map=None, + **kwargs, ): """ Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline) @@ -613,6 +619,8 @@ def train_new_from_iterator( if you have everything in memory. vocab_size (`int`): The size of the vocabulary you want for your tokenizer. + length (`int`, *optional*): + The total number of sequences in the iterator. This is used to provide meaningful progress tracking new_special_tokens (list of `str` or `AddedToken`, *optional*): A list of new special tokens to add to the tokenizer you are training. special_tokens_map (`Dict[str, str]`, *optional*): @@ -694,7 +702,7 @@ def train_new_from_iterator( trainer_class = MODEL_TO_TRAINER_MAPPING[tokenizer_json["model"]["type"]] trainer = trainer_class(vocab_size=vocab_size, special_tokens=special_tokens, **kwargs) - tokenizer.train_from_iterator(text_iterator, trainer=trainer) + tokenizer.train_from_iterator(text_iterator, length=length, trainer=trainer) if post_processor is not None: trained_tokenizer_json = json.loads(tokenizer.to_str()) From ae189ef99199d07bc7c6dfb79c46b411e05dfe61 Mon Sep 17 00:00:00 2001 From: tomerip <59172869+tomerip@users.noreply.github.com> Date: Wed, 30 Mar 2022 18:56:03 +0300 Subject: [PATCH 25/54] Add support for exporting GPT-J to ONNX-TRT (#16492) Add support for exporting GPT-J to ONNX-TRT Co-authored-by: Tomer Stav --- src/transformers/models/gptj/modeling_gptj.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index b957cec54a8e..f45631322050 100755 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -62,8 +62,19 @@ def rotate_every_two(x): return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)') +def duplicate_interleave(m): + """ + A simple version of `torch.repeat_interleave` for duplicating a matrix while interleaving the copy. + """ + dim0 = m.shape[0] + m = m.view(-1, 1) # flatten the matrix + m = m.repeat(1, 2) # repeat all elements into the 2nd dimension + m = m.view(dim0, -1) # reshape into a matrix, interleaving the copy + return m + + def apply_rotary_pos_emb(x, sincos, offset=0): - sin, cos = map(lambda t: t[None, offset : x.shape[1] + offset, None, :].repeat_interleave(2, 3), sincos) + sin, cos = map(lambda t: duplicate_interleave(t)[None, offset : x.shape[1] + offset, None, :], sincos) # einsum notation for lambda t: repeat(t[offset:x.shape[1]+offset,:], "n d -> () n () (d j)", j=2) return (x * cos) + (rotate_every_two(x) * sin) From c2f8eaf6bcb3a8a3f25289726bdfec880cf178cc Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 30 Mar 2022 17:12:27 +0100 Subject: [PATCH 26/54] TF: unpack inputs on Convbert, GPTJ, LED, and templates (#16491) * Add unpack_inputs to remaining models * remove stray use of inputs in the templates; fix tf.debugging of attn masks --- .../models/bart/modeling_tf_bart.py | 4 +- .../models/convbert/modeling_tf_convbert.py | 254 ++----- .../models/gptj/modeling_tf_gptj.py | 111 ++- .../models/led/modeling_tf_led.py | 366 ++++------ .../models/mbart/modeling_tf_mbart.py | 4 +- .../modeling_tf_speech_to_text.py | 4 +- ...tf_{{cookiecutter.lowercase_modelname}}.py | 656 ++++++------------ 7 files changed, 466 insertions(+), 933 deletions(-) diff --git a/src/transformers/models/bart/modeling_tf_bart.py b/src/transformers/models/bart/modeling_tf_bart.py index 9599bfe1d1e2..106a87c043c9 100644 --- a/src/transformers/models/bart/modeling_tf_bart.py +++ b/src/transformers/models/bart/modeling_tf_bart.py @@ -943,12 +943,12 @@ def call( # check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired # The tf.debugging asserts are not compliant with XLA then they # have to be disabled in other modes than eager. - for attn_mask in [head_mask, cross_attn_head_mask]: + for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]: if attn_mask is not None and tf.executing_eagerly(): tf.debugging.assert_equal( shape_list(attn_mask)[0], len(self.layers), - message=f"The {attn_mask} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", + message=f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", ) for idx, decoder_layer in enumerate(self.layers): diff --git a/src/transformers/models/convbert/modeling_tf_convbert.py b/src/transformers/models/convbert/modeling_tf_convbert.py index 61a4f5d69e42..f167325527b6 100644 --- a/src/transformers/models/convbert/modeling_tf_convbert.py +++ b/src/transformers/models/convbert/modeling_tf_convbert.py @@ -39,8 +39,8 @@ TFSequenceSummary, TFTokenClassificationLoss, get_initializer, - input_processing, keras_serializable, + unpack_inputs, ) from ...tf_utils import shape_list from ...utils import ( @@ -568,6 +568,7 @@ def get_head_mask(self, head_mask): return head_mask + @unpack_inputs def call( self, input_ids=None, @@ -582,60 +583,36 @@ def call( training=False, **kwargs, ): - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") - if inputs["attention_mask"] is None: - inputs["attention_mask"] = tf.fill(input_shape, 1) + if attention_mask is None: + attention_mask = tf.fill(input_shape, 1) - if inputs["token_type_ids"] is None: - inputs["token_type_ids"] = tf.fill(input_shape, 0) + if token_type_ids is None: + token_type_ids = tf.fill(input_shape, 0) - hidden_states = self.embeddings( - inputs["input_ids"], - inputs["position_ids"], - inputs["token_type_ids"], - inputs["inputs_embeds"], - training=inputs["training"], - ) - extended_attention_mask = self.get_extended_attention_mask( - inputs["attention_mask"], input_shape, hidden_states.dtype - ) - inputs["head_mask"] = self.get_head_mask(inputs["head_mask"]) + hidden_states = self.embeddings(input_ids, position_ids, token_type_ids, inputs_embeds, training=training) + extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape, hidden_states.dtype) + head_mask = self.get_head_mask(head_mask) if hasattr(self, "embeddings_project"): - hidden_states = self.embeddings_project(hidden_states, training=inputs["training"]) + hidden_states = self.embeddings_project(hidden_states, training=training) hidden_states = self.encoder( hidden_states, extended_attention_mask, - inputs["head_mask"], - inputs["output_attentions"], - inputs["output_hidden_states"], - inputs["return_dict"], - training=inputs["training"], + head_mask, + output_attentions, + output_hidden_states, + return_dict, + training=training, ) return hidden_states @@ -754,6 +731,7 @@ def __init__(self, config, *inputs, **kwargs): self.convbert = TFConvBertMainLayer(config, name="convbert") + @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -775,9 +753,7 @@ def call( training=False, **kwargs, ): - inputs = input_processing( - func=self.call, - config=self.config, + outputs = self.convbert( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -788,19 +764,6 @@ def call( output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - outputs = self.convbert( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) return outputs @@ -886,6 +849,7 @@ def get_lm_head(self): def get_prefix_bias_name(self): return self.name + "/" + self.generator_lm_head.name + @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -914,9 +878,7 @@ def call( config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` """ - inputs = input_processing( - func=self.call, - config=self.config, + generator_hidden_states = self.convbert( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -926,28 +888,14 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - generator_hidden_states = self.convbert( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) generator_sequence_output = generator_hidden_states[0] - prediction_scores = self.generator_predictions(generator_sequence_output, training=inputs["training"]) - prediction_scores = self.generator_lm_head(prediction_scores, training=inputs["training"]) - loss = None if inputs["labels"] is None else self.hf_compute_loss(inputs["labels"], prediction_scores) + prediction_scores = self.generator_predictions(generator_sequence_output, training=training) + prediction_scores = self.generator_lm_head(prediction_scores, training=training) + loss = None if labels is None else self.hf_compute_loss(labels, prediction_scores) - if not inputs["return_dict"]: + if not return_dict: output = (prediction_scores,) + generator_hidden_states[1:] return ((loss,) + output) if loss is not None else output @@ -1010,6 +958,7 @@ def __init__(self, config, *inputs, **kwargs): self.convbert = TFConvBertMainLayer(config, name="convbert") self.classifier = TFConvBertClassificationHead(config, name="classifier") + @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1038,10 +987,8 @@ def call( config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, + outputs = self.convbert( + input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, @@ -1050,26 +997,12 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, ) - outputs = self.convbert( - inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], - ) - logits = self.classifier(outputs[0], training=inputs["training"]) - loss = None if inputs["labels"] is None else self.hf_compute_loss(inputs["labels"], logits) + logits = self.classifier(outputs[0], training=training) + loss = None if labels is None else self.hf_compute_loss(labels, logits) - if not inputs["return_dict"]: + if not return_dict: output = (logits,) + outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1117,6 +1050,7 @@ def dummy_inputs(self): """ return {"input_ids": tf.convert_to_tensor(MULTIPLE_CHOICE_DUMMY_INPUTS)} + @unpack_inputs @add_start_docstrings_to_model_forward( CONVBERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length") ) @@ -1146,43 +1080,20 @@ def call( Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - labels=labels, - training=training, - kwargs_call=kwargs, - ) - - if inputs["input_ids"] is not None: - num_choices = shape_list(inputs["input_ids"])[1] - seq_length = shape_list(inputs["input_ids"])[2] + if input_ids is not None: + num_choices = shape_list(input_ids)[1] + seq_length = shape_list(input_ids)[2] else: - num_choices = shape_list(inputs["inputs_embeds"])[1] - seq_length = shape_list(inputs["inputs_embeds"])[2] + num_choices = shape_list(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[2] - flat_input_ids = tf.reshape(inputs["input_ids"], (-1, seq_length)) if inputs["input_ids"] is not None else None - flat_attention_mask = ( - tf.reshape(inputs["attention_mask"], (-1, seq_length)) if inputs["attention_mask"] is not None else None - ) - flat_token_type_ids = ( - tf.reshape(inputs["token_type_ids"], (-1, seq_length)) if inputs["token_type_ids"] is not None else None - ) - flat_position_ids = ( - tf.reshape(inputs["position_ids"], (-1, seq_length)) if inputs["position_ids"] is not None else None - ) + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None + flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None + flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None + flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None flat_inputs_embeds = ( - tf.reshape(inputs["inputs_embeds"], (-1, seq_length, shape_list(inputs["inputs_embeds"])[3])) - if inputs["inputs_embeds"] is not None + tf.reshape(inputs_embeds, (-1, seq_length, shape_list(inputs_embeds)[3])) + if inputs_embeds is not None else None ) outputs = self.convbert( @@ -1190,19 +1101,19 @@ def call( flat_attention_mask, flat_token_type_ids, flat_position_ids, - inputs["head_mask"], + head_mask, flat_inputs_embeds, - inputs["output_attentions"], - inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], + output_attentions, + output_hidden_states, + return_dict=return_dict, + training=training, ) - logits = self.sequence_summary(outputs[0], training=inputs["training"]) + logits = self.sequence_summary(outputs[0], training=training) logits = self.classifier(logits) reshaped_logits = tf.reshape(logits, (-1, num_choices)) - loss = None if inputs["labels"] is None else self.hf_compute_loss(inputs["labels"], reshaped_logits) + loss = None if labels is None else self.hf_compute_loss(labels, reshaped_logits) - if not inputs["return_dict"]: + if not return_dict: output = (reshaped_logits,) + outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1256,6 +1167,7 @@ def __init__(self, config, *inputs, **kwargs): config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="classifier" ) + @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1282,10 +1194,8 @@ def call( labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, + outputs = self.convbert( + input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, @@ -1294,28 +1204,14 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - outputs = self.convbert( - inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) sequence_output = outputs[0] - sequence_output = self.dropout(sequence_output, training=inputs["training"]) + sequence_output = self.dropout(sequence_output, training=training) logits = self.classifier(sequence_output) - loss = None if inputs["labels"] is None else self.hf_compute_loss(inputs["labels"], logits) + loss = None if labels is None else self.hf_compute_loss(labels, logits) - if not inputs["return_dict"]: + if not return_dict: output = (logits,) + outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1350,6 +1246,7 @@ def __init__(self, config, *inputs, **kwargs): config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="qa_outputs" ) + @unpack_inputs @add_start_docstrings_to_model_forward(CONVBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1383,10 +1280,8 @@ def call( Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, + outputs = self.convbert( + input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, position_ids=position_ids, @@ -1395,22 +1290,7 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - start_positions=start_positions, - end_positions=end_positions, training=training, - kwargs_call=kwargs, - ) - outputs = self.convbert( - inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) sequence_output = outputs[0] logits = self.qa_outputs(sequence_output) @@ -1419,12 +1299,12 @@ def call( end_logits = tf.squeeze(end_logits, axis=-1) loss = None - if inputs["start_positions"] is not None and inputs["end_positions"] is not None: - labels = {"start_position": inputs["start_positions"]} - labels["end_position"] = inputs["end_positions"] + if start_positions is not None and end_positions is not None: + labels = {"start_position": start_positions} + labels["end_position"] = end_positions loss = self.hf_compute_loss(labels, (start_logits, end_logits)) - if not inputs["return_dict"]: + if not return_dict: output = (start_logits, end_logits) + outputs[1:] return ((loss,) + output) if loss is not None else output diff --git a/src/transformers/models/gptj/modeling_tf_gptj.py b/src/transformers/models/gptj/modeling_tf_gptj.py index e6d9a4f308ac..ce5c5d78e5ae 100644 --- a/src/transformers/models/gptj/modeling_tf_gptj.py +++ b/src/transformers/models/gptj/modeling_tf_gptj.py @@ -40,7 +40,6 @@ TFSequenceClassificationLoss, TFSharedEmbeddings, get_initializer, - input_processing, keras_serializable, unpack_inputs, ) @@ -376,6 +375,7 @@ def _prune_heads(self, heads_to_prune): """ raise NotImplementedError + @unpack_inputs def call( self, input_ids=None, @@ -392,53 +392,34 @@ def call( training=False, **kwargs, ): - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - past_key_values=past_key_values, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - inputs["input_ids"] = tf.reshape(inputs["input_ids"], [-1, input_shape[-1]]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") - if inputs["past_key_values"] is None: + if past_key_values is None: past_length = 0 - inputs["past_key_values"] = [None] * len(self.h) + past_key_values = [None] * len(self.h) else: - past_length = shape_list(inputs["past_key_values"][0][0])[-2] + past_length = shape_list(past_key_values[0][0])[-2] - if inputs["position_ids"] is None: - inputs["position_ids"] = tf.expand_dims(tf.range(past_length, input_shape[-1] + past_length), axis=0) + if position_ids is None: + position_ids = tf.expand_dims(tf.range(past_length, input_shape[-1] + past_length), axis=0) - if inputs["attention_mask"] is not None: + if attention_mask is not None: # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] # this attention mask is more simple than the triangular masking of causal attention # used in OpenAI GPT, we just need to prepare the broadcast dimension here. - attention_mask_shape = shape_list(inputs["attention_mask"]) - inputs["attention_mask"] = tf.reshape( - inputs["attention_mask"], (attention_mask_shape[0], 1, 1, attention_mask_shape[1]) - ) + attention_mask_shape = shape_list(attention_mask) + attention_mask = tf.reshape(attention_mask, (attention_mask_shape[0], 1, 1, attention_mask_shape[1])) # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for @@ -446,78 +427,74 @@ def call( # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. one_cst = tf.constant(1.0) - inputs["attention_mask"] = tf.cast(inputs["attention_mask"], dtype=one_cst.dtype) - inputs["attention_mask"] = tf.multiply( - tf.subtract(one_cst, inputs["attention_mask"]), tf.constant(-10000.0) - ) + attention_mask = tf.cast(attention_mask, dtype=one_cst.dtype) + attention_mask = tf.multiply(tf.subtract(one_cst, attention_mask), tf.constant(-10000.0)) # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head # attention_probs has shape bsz x n_heads x N x N # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] - if inputs["head_mask"] is not None: + if head_mask is not None: raise NotImplementedError else: - inputs["head_mask"] = [None] * self.num_hidden_layers + head_mask = [None] * self.num_hidden_layers # head_mask = tf.constant([0] * self.num_hidden_layers) - inputs["position_ids"] = tf.reshape(inputs["position_ids"], [-1, shape_list(inputs["position_ids"])[-1]]) + position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]]) - if inputs["inputs_embeds"] is None: - inputs["inputs_embeds"] = self.wte(inputs["input_ids"], mode="embedding") + if inputs_embeds is None: + inputs_embeds = self.wte(input_ids, mode="embedding") - if inputs["token_type_ids"] is not None: - inputs["token_type_ids"] = tf.reshape( - inputs["token_type_ids"], [-1, shape_list(inputs["token_type_ids"])[-1]] - ) - token_type_embeds = self.wte(inputs["token_type_ids"], mode="embedding") + if token_type_ids is not None: + token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]]) + token_type_embeds = self.wte(token_type_ids, mode="embedding") else: token_type_embeds = tf.constant(0.0) - token_type_embeds = tf.cast(token_type_embeds, dtype=inputs["inputs_embeds"].dtype) - hidden_states = inputs["inputs_embeds"] + token_type_embeds - hidden_states = self.drop(hidden_states, training=inputs["training"]) + token_type_embeds = tf.cast(token_type_embeds, dtype=inputs_embeds.dtype) + hidden_states = inputs_embeds + token_type_embeds + hidden_states = self.drop(hidden_states, training=training) output_shape = input_shape + [shape_list(hidden_states)[-1]] - presents = () if inputs["use_cache"] else None - all_attentions = () if inputs["output_attentions"] else None - all_hidden_states = () if inputs["output_hidden_states"] else None - for i, (block, layer_past) in enumerate(zip(self.h, inputs["past_key_values"])): - if inputs["output_hidden_states"]: + presents = () if use_cache else None + all_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)): + if output_hidden_states: all_hidden_states = all_hidden_states + (tf.reshape(hidden_states, output_shape),) outputs = block( hidden_states, layer_past, - inputs["attention_mask"], - inputs["head_mask"][i], - inputs["use_cache"], - inputs["output_attentions"], - training=inputs["training"], + attention_mask, + head_mask[i], + use_cache, + output_attentions, + training=training, ) hidden_states = outputs[0] - if inputs["use_cache"]: + if use_cache: presents = presents + (outputs[1],) - if inputs["output_attentions"]: - all_attentions = all_attentions + (outputs[2 if inputs["use_cache"] else 1],) + if output_attentions: + all_attentions = all_attentions + (outputs[2 if use_cache else 1],) hidden_states = self.ln_f(hidden_states) hidden_states = tf.reshape(hidden_states, output_shape) # Add last hidden state - if inputs["output_hidden_states"]: + if output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) - if inputs["output_attentions"]: + if output_attentions: # let the number of heads free (-1) so we can extract attention even after head pruning attention_output_shape = input_shape[:-1] + [-1] + shape_list(all_attentions[0])[-2:] all_attentions = tuple(tf.reshape(t, attention_output_shape) for t in all_attentions) - if not inputs["return_dict"]: + if not return_dict: return tuple(v for v in [hidden_states, presents, all_hidden_states, all_attentions] if v is not None) return TFBaseModelOutputWithPast( diff --git a/src/transformers/models/led/modeling_tf_led.py b/src/transformers/models/led/modeling_tf_led.py index cdd82c6c5058..4519f5df9808 100644 --- a/src/transformers/models/led/modeling_tf_led.py +++ b/src/transformers/models/led/modeling_tf_led.py @@ -30,8 +30,8 @@ TFSharedEmbeddings, TFWrappedEmbeddings, get_initializer, - input_processing, keras_serializable, + unpack_inputs, ) from ...tf_utils import shape_list from ...utils import ( @@ -1654,6 +1654,7 @@ def get_embed_tokens(self): def set_embed_tokens(self, embed_tokens): self.embed_tokens = embed_tokens + @unpack_inputs def call( self, input_ids=None, @@ -1703,95 +1704,74 @@ def call( return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - head_mask=head_mask, - global_attention_mask=global_attention_mask, - inputs_embeds=inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - inputs["inputs_embeds"] = self.embed_tokens(inputs["input_ids"]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + inputs_embeds = self.embed_tokens(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") - if inputs["attention_mask"] is None: - inputs["attention_mask"] = tf.fill(input_shape, 1) + if attention_mask is None: + attention_mask = tf.fill(input_shape, 1) # merge `global_attention_mask` and `attention_mask` - if inputs["global_attention_mask"] is not None: - inputs["attention_mask"] = inputs["attention_mask"] * tf.cast( - (inputs["global_attention_mask"] + 1), dtype=inputs["attention_mask"].dtype - ) + if global_attention_mask is not None: + attention_mask = attention_mask * tf.cast((global_attention_mask + 1), dtype=attention_mask.dtype) - ( - padding_len, - inputs["input_ids"], - inputs["attention_mask"], - inputs["inputs_embeds"], - ) = self._pad_to_window_size( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - inputs_embeds=inputs["inputs_embeds"], + (padding_len, input_ids, attention_mask, inputs_embeds,) = self._pad_to_window_size( + input_ids=input_ids, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, pad_token_id=self.padding_idx, ) - input_shape = shape_list(inputs["attention_mask"]) + input_shape = shape_list(attention_mask) # is index masked or global attention - is_index_masked = tf.math.less(tf.cast(inputs["attention_mask"], tf.int8), 1) - is_index_global_attn = tf.math.greater(tf.cast(inputs["attention_mask"], tf.int8), 1) + is_index_masked = tf.math.less(tf.cast(attention_mask, tf.int8), 1) + is_index_global_attn = tf.math.greater(tf.cast(attention_mask, tf.int8), 1) is_global_attn = tf.math.reduce_any(is_index_global_attn) embed_pos = self.embed_positions(input_shape) - hidden_states = inputs["inputs_embeds"] + embed_pos + hidden_states = inputs_embeds + embed_pos hidden_states = self.layernorm_embedding(hidden_states) - hidden_states = self.dropout(hidden_states, training=inputs["training"]) + hidden_states = self.dropout(hidden_states, training=training) # check attention mask and invert - if inputs["attention_mask"] is not None: + if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - inputs["attention_mask"] = _expand_mask(inputs["attention_mask"])[:, 0, 0, :] - inputs["attention_mask"] = inputs["attention_mask"][:, :, None, None] + attention_mask = _expand_mask(attention_mask)[:, 0, 0, :] + attention_mask = attention_mask[:, :, None, None] - encoder_states = () if inputs["output_hidden_states"] else None - all_attentions = all_global_attentions = () if inputs["output_attentions"] else None + encoder_states = () if output_hidden_states else None + all_attentions = all_global_attentions = () if output_attentions else None # check if head_mask has a correct number of layers specified if desired - if inputs["head_mask"] is not None and tf.executing_eagerly(): + if head_mask is not None and tf.executing_eagerly(): tf.debugging.assert_equal( - shape_list(inputs["head_mask"])[0], + shape_list(head_mask)[0], len(self.layers), - message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(inputs['head_mask'])[0]}.", + message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(head_mask)[0]}.", ) # encoder layers for idx, encoder_layer in enumerate(self.layers): - if inputs["output_hidden_states"]: + if output_hidden_states: hidden_states_to_add = self.compute_hidden_states(hidden_states, padding_len) encoder_states = encoder_states + (hidden_states_to_add,) # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) dropout_probability = random.uniform(0, 1) - if inputs["training"] and (dropout_probability < self.layerdrop): # skip the layer + if training and (dropout_probability < self.layerdrop): # skip the layer continue layer_outputs = encoder_layer( hidden_states=hidden_states, - attention_mask=inputs["attention_mask"], - layer_head_mask=inputs["head_mask"][idx] if inputs["head_mask"] is not None else None, + attention_mask=attention_mask, + layer_head_mask=head_mask[idx] if head_mask is not None else None, is_index_masked=is_index_masked, is_index_global_attn=is_index_global_attn, is_global_attn=is_global_attn, @@ -1799,7 +1779,7 @@ def call( hidden_states = layer_outputs[0] - if inputs["output_attentions"]: + if output_attentions: # bzs x seq_len x num_attn_heads x (num_global_attn + attention_window_len + 1) => bzs x num_attn_heads x seq_len x (num_global_attn + attention_window_len + 1) all_attentions = all_attentions + (tf.transpose(layer_outputs[1], (0, 2, 1, 3)),) @@ -1811,17 +1791,17 @@ def call( hidden_states = self.compute_hidden_states(hidden_states, padding_len) # undo padding - if inputs["output_attentions"]: + if output_attentions: all_attentions = ( tuple([state[:, :, :-padding_len, :] for state in all_attentions]) if padding_len > 0 else all_attentions ) - if inputs["output_hidden_states"]: + if output_hidden_states: encoder_states = encoder_states + (hidden_states,) - if not inputs["return_dict"]: + if not return_dict: return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) return TFLEDEncoderBaseModelOutput( last_hidden_state=hidden_states, @@ -1915,6 +1895,7 @@ def __init__(self, config: LEDConfig, embed_tokens: Optional[TFSharedEmbeddings] def set_embed_tokens(self, embed_tokens): self.embed_tokens = embed_tokens + @unpack_inputs def call( self, input_ids=None, @@ -1985,45 +1966,25 @@ def call( return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - head_mask=head_mask, - encoder_head_mask=encoder_head_mask, - inputs_embeds=inputs_embeds, - past_key_values=past_key_values, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") - past_key_values_length = ( - shape_list(inputs["past_key_values"][0][0])[2] if inputs["past_key_values"] is not None else 0 - ) + past_key_values_length = shape_list(past_key_values[0][0])[2] if past_key_values is not None else 0 # embed positions positions = self.embed_positions(input_shape, past_key_values_length) - if inputs["inputs_embeds"] is None: - inputs["inputs_embeds"] = self.embed_tokens(inputs["input_ids"]) + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) - hidden_states = inputs["inputs_embeds"] + hidden_states = inputs_embeds # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] if input_shape[-1] > 1: @@ -2033,17 +1994,15 @@ def call( tf.ones((input_shape[0], input_shape[1] + past_key_values_length)), tgt_len=input_shape[-1] ) - if inputs["attention_mask"] is not None and input_shape[-1] > 1: - combined_attention_mask = combined_attention_mask + _expand_mask( - inputs["attention_mask"], tgt_len=input_shape[-1] - ) + if attention_mask is not None and input_shape[-1] > 1: + combined_attention_mask = combined_attention_mask + _expand_mask(attention_mask, tgt_len=input_shape[-1]) - if inputs["encoder_hidden_states"] is not None and inputs["encoder_attention_mask"] is not None: + if encoder_hidden_states is not None and encoder_attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - inputs["encoder_attention_mask"] = _expand_mask(inputs["encoder_attention_mask"], tgt_len=input_shape[-1]) + encoder_attention_mask = _expand_mask(encoder_attention_mask, tgt_len=input_shape[-1]) hidden_states = self.layernorm_embedding(hidden_states + positions) - hidden_states = self.dropout(hidden_states, training=inputs["training"]) + hidden_states = self.dropout(hidden_states, training=training) # decoder layers all_hidden_states = () @@ -2052,54 +2011,52 @@ def call( present_key_values = () # check if head_mask has a correct number of layers specified if desired - if inputs["head_mask"] is not None and tf.executing_eagerly(): + if head_mask is not None and tf.executing_eagerly(): tf.debugging.assert_equal( - shape_list(inputs["head_mask"])[0], + shape_list(head_mask)[0], len(self.layers), - message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(inputs['head_mask'])[0]}.", + message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(head_mask)[0]}.", ) for idx, decoder_layer in enumerate(self.layers): # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) - if inputs["output_hidden_states"]: + if output_hidden_states: all_hidden_states += (hidden_states,) dropout_probability = random.uniform(0, 1) - if inputs["training"] and (dropout_probability < self.layerdrop): + if training and (dropout_probability < self.layerdrop): continue - past_key_value = inputs["past_key_values"][idx] if inputs["past_key_values"] is not None else None + past_key_value = past_key_values[idx] if past_key_values is not None else None hidden_states, layer_self_attn, layer_cross_attn, present_key_value = decoder_layer( hidden_states, attention_mask=combined_attention_mask, - encoder_hidden_states=inputs["encoder_hidden_states"], - encoder_attention_mask=inputs["encoder_attention_mask"], - layer_head_mask=inputs["head_mask"][idx] if inputs["head_mask"] is not None else None, - encoder_layer_head_mask=inputs["encoder_head_mask"][idx] - if inputs["encoder_head_mask"] is not None - else None, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + layer_head_mask=head_mask[idx] if head_mask is not None else None, + encoder_layer_head_mask=encoder_head_mask[idx] if encoder_head_mask is not None else None, past_key_value=past_key_value, ) - if inputs["use_cache"]: + if use_cache: present_key_values += (present_key_value,) - if inputs["output_attentions"]: + if output_attentions: all_self_attns += (layer_self_attn,) all_cross_attentions += (layer_cross_attn,) - if inputs["output_hidden_states"]: + if output_hidden_states: all_hidden_states += (hidden_states,) else: all_hidden_states = None - all_self_attns = all_self_attns if inputs["output_attentions"] else None - all_cross_attentions = all_cross_attentions if inputs["output_attentions"] else None + all_self_attns = all_self_attns if output_attentions else None + all_cross_attentions = all_cross_attentions if output_attentions else None - present_key_values = present_key_values if inputs["use_cache"] else None + present_key_values = present_key_values if use_cache else None - if not inputs["return_dict"]: + if not return_dict: return tuple( v for v in [hidden_states, present_key_values, all_hidden_states, all_self_attns, all_cross_attentions] @@ -2149,6 +2106,7 @@ def set_input_embeddings(self, new_embeddings): self.encoder.set_embed_tokens(embed_tokens) self.decoder.set_embed_tokens(embed_tokens) + @unpack_inputs def call( self, input_ids=None, @@ -2169,72 +2127,51 @@ def call( training=False, **kwargs ): - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, - encoder_outputs=encoder_outputs, - global_attention_mask=global_attention_mask, + + if decoder_input_ids is None and decoder_inputs_embeds is None: + use_cache = False + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + global_attention_mask=global_attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a TFLEDEncoderBaseModelOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, TFLEDEncoderBaseModelOutput): + encoder_outputs = TFLEDEncoderBaseModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + ) + # If the user passed a TFLEDEncoderBaseModelOutput for encoder_outputs, we wrap it in a tuple when return_dict=False + elif not return_dict and not isinstance(encoder_outputs, tuple): + encoder_outputs = encoder_outputs.to_tuple() + + decoder_outputs = self.decoder( + decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_outputs[0], + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + encoder_head_mask=head_mask, past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, + inputs_embeds=decoder_inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - - if inputs["decoder_input_ids"] is None and inputs["decoder_inputs_embeds"] is None: - inputs["use_cache"] = False - - if inputs["encoder_outputs"] is None: - inputs["encoder_outputs"] = self.encoder( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - global_attention_mask=inputs["global_attention_mask"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], - ) - # If the user passed a tuple for encoder_outputs, we wrap it in a TFLEDEncoderBaseModelOutput when return_dict=True - elif inputs["return_dict"] and not isinstance(inputs["encoder_outputs"], TFLEDEncoderBaseModelOutput): - inputs["encoder_outputs"] = TFLEDEncoderBaseModelOutput( - last_hidden_state=inputs["encoder_outputs"][0], - hidden_states=inputs["encoder_outputs"][1] if len(inputs["encoder_outputs"]) > 1 else None, - attentions=inputs["encoder_outputs"][2] if len(inputs["encoder_outputs"]) > 2 else None, - ) - # If the user passed a TFLEDEncoderBaseModelOutput for encoder_outputs, we wrap it in a tuple when return_dict=False - elif not inputs["return_dict"] and not isinstance(inputs["encoder_outputs"], tuple): - inputs["encoder_outputs"] = inputs["encoder_outputs"].to_tuple() + ) - decoder_outputs = self.decoder( - inputs["decoder_input_ids"], - attention_mask=inputs["decoder_attention_mask"], - encoder_hidden_states=inputs["encoder_outputs"][0], - encoder_attention_mask=inputs["attention_mask"], - head_mask=inputs["decoder_head_mask"], - encoder_head_mask=inputs["head_mask"], - past_key_values=inputs["past_key_values"], - inputs_embeds=inputs["decoder_inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], - ) - - if not inputs["return_dict"]: - return decoder_outputs + inputs["encoder_outputs"] + if not return_dict: + return decoder_outputs + encoder_outputs return TFLEDSeq2SeqModelOutput( last_hidden_state=decoder_outputs.last_hidden_state, @@ -2242,10 +2179,10 @@ def call( decoder_hidden_states=decoder_outputs.hidden_states, decoder_attentions=decoder_outputs.attentions, cross_attentions=decoder_outputs.cross_attentions, - encoder_last_hidden_state=inputs["encoder_outputs"].last_hidden_state, - encoder_hidden_states=inputs["encoder_outputs"].hidden_states, - encoder_attentions=inputs["encoder_outputs"].attentions, - encoder_global_attentions=inputs["encoder_outputs"].global_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + encoder_global_attentions=encoder_outputs.global_attentions, ) @@ -2265,6 +2202,7 @@ def get_encoder(self): def get_decoder(self): return self.led.decoder + @unpack_inputs @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -2292,17 +2230,16 @@ def call( training=False, **kwargs ): - inputs = input_processing( - func=self.call, - config=self.config, + + outputs = self.led( input_ids=input_ids, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, encoder_outputs=encoder_outputs, global_attention_mask=global_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, @@ -2311,25 +2248,6 @@ def call( output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - outputs = self.led( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - decoder_input_ids=inputs["decoder_input_ids"], - decoder_attention_mask=inputs["decoder_attention_mask"], - encoder_outputs=inputs["encoder_outputs"], - global_attention_mask=inputs["global_attention_mask"], - head_mask=inputs["head_mask"], - decoder_head_mask=inputs["decoder_head_mask"], - past_key_values=inputs["past_key_values"], - inputs_embeds=inputs["inputs_embeds"], - decoder_inputs_embeds=inputs["decoder_inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) return outputs @@ -2393,6 +2311,7 @@ def get_output_embeddings(self): def set_output_embeddings(self, value): self.set_input_embeddings(value) + @unpack_inputs @add_start_docstrings_to_model_forward(LED_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=TFLEDSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) def call( @@ -2435,17 +2354,22 @@ def call( >>> # probs[5] is associated with the mask token ```""" - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, + if labels is not None: + use_cache = False + if decoder_input_ids is None: + decoder_input_ids = shift_tokens_right( + labels, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + outputs = self.led( + input_ids, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, encoder_outputs=encoder_outputs, global_attention_mask=global_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, @@ -2453,41 +2377,13 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - - if inputs["labels"] is not None: - inputs["use_cache"] = False - if inputs["decoder_input_ids"] is None: - inputs["decoder_input_ids"] = shift_tokens_right( - inputs["labels"], self.config.pad_token_id, self.config.decoder_start_token_id - ) - - outputs = self.led( - inputs["input_ids"], - attention_mask=inputs["attention_mask"], - decoder_input_ids=inputs["decoder_input_ids"], - decoder_attention_mask=inputs["decoder_attention_mask"], - encoder_outputs=inputs["encoder_outputs"], - global_attention_mask=inputs["global_attention_mask"], - head_mask=inputs["head_mask"], - decoder_head_mask=inputs["decoder_head_mask"], - past_key_values=inputs["past_key_values"], - inputs_embeds=inputs["inputs_embeds"], - decoder_inputs_embeds=inputs["decoder_inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) lm_logits = self.led.shared(outputs[0], mode="linear") lm_logits = lm_logits + self.final_logits_bias - masked_lm_loss = None if inputs["labels"] is None else self.hf_compute_loss(inputs["labels"], lm_logits) + masked_lm_loss = None if labels is None else self.hf_compute_loss(labels, lm_logits) - if not inputs["return_dict"]: + if not return_dict: output = (lm_logits,) + outputs[1:] return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output return TFLEDSeq2SeqLMOutput( diff --git a/src/transformers/models/mbart/modeling_tf_mbart.py b/src/transformers/models/mbart/modeling_tf_mbart.py index f9f3014c67ff..3f2ea655f455 100644 --- a/src/transformers/models/mbart/modeling_tf_mbart.py +++ b/src/transformers/models/mbart/modeling_tf_mbart.py @@ -965,12 +965,12 @@ def call( # check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired # The tf.debugging asserts are not compliant with XLA then they # have to be disabled in other modes than eager. - for attn_mask in [head_mask, cross_attn_head_mask]: + for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]: if attn_mask is not None and tf.executing_eagerly(): tf.debugging.assert_equal( shape_list(attn_mask)[0], len(self.layers), - message=f"The {attn_mask} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", + message=f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", ) for idx, decoder_layer in enumerate(self.layers): diff --git a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py index 7bdf620c6555..6c78ab1b58f3 100755 --- a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py @@ -1060,12 +1060,12 @@ def call( # check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired # The tf.debugging asserts are not compliant with XLA then they have to be disabled in other modes than eager. - for attn_mask in [head_mask, cross_attn_head_mask]: + for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]: if attn_mask is not None and tf.executing_eagerly(): tf.debugging.assert_equal( shape_list(attn_mask)[0], len(self.layers), - message=f"The {attn_mask} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", + message=f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", ) for idx, decoder_layer in enumerate(self.layers): diff --git a/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py b/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py index 0fd552e02905..da2a0a3828f9 100644 --- a/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py +++ b/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py @@ -50,8 +50,8 @@ TFSequenceSummary, TFTokenClassificationLoss, get_initializer, - input_processing, keras_serializable, + unpack_inputs, ) from ...tf_utils import shape_list from ...utils import logging @@ -636,6 +636,7 @@ class PreTrainedModel """ raise NotImplementedError + @unpack_inputs def call( self, input_ids: Optional[TFModelInputType] = None, @@ -654,59 +655,40 @@ def call( training: bool = False, **kwargs, ) -> Union[TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor]]: - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - past_key_values=past_key_values, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) if not self.config.is_decoder: - inputs["use_cache"] = False + use_cache = False - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") batch_size, seq_length = input_shape - if inputs["past_key_values"] is None: + if past_key_values is None: past_key_values_length = 0 - inputs["past_key_values"] = [None] * len(self.encoder.layer) + past_key_values = [None] * len(self.encoder.layer) else: - past_key_values_length = shape_list(inputs["past_key_values"][0][0])[-2] + past_key_values_length = shape_list(past_key_values[0][0])[-2] - if inputs["attention_mask"] is None: - inputs["attention_mask"] = tf.fill(dims=(batch_size, seq_length + past_key_values_length), value=1) + if attention_mask is None: + attention_mask = tf.fill(dims=(batch_size, seq_length + past_key_values_length), value=1) - if inputs["token_type_ids"] is None: - inputs["token_type_ids"] = tf.fill(dims=input_shape, value=0) + if token_type_ids is None: + token_type_ids = tf.fill(dims=input_shape, value=0) embedding_output = self.embeddings( - input_ids=inputs["input_ids"], - position_ids=inputs["position_ids"], - token_type_ids=inputs["token_type_ids"], - inputs_embeds=inputs["inputs_embeds"], + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, past_key_values_length=past_key_values_length, - training=inputs["training"], + training=training, ) # We create a 3D attention mask from a 2D tensor mask. @@ -714,7 +696,7 @@ def call( # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] # this attention mask is more simple than the triangular masking of causal attention # used in OpenAI GPT, we just need to prepare the broadcast dimension here. - attention_mask_shape = shape_list(inputs["attention_mask"]) + attention_mask_shape = shape_list(attention_mask) mask_seq_length = seq_length + past_key_values_length # Copied from `modeling_tf_t5.py` @@ -727,18 +709,18 @@ def call( tf.tile(seq_ids[None, None, :], (batch_size, mask_seq_length, 1)), seq_ids[None, :, None], ) - causal_mask = tf.cast(causal_mask, dtype=inputs["attention_mask"].dtype) - extended_attention_mask = causal_mask * inputs["attention_mask"][:, None, :] + causal_mask = tf.cast(causal_mask, dtype=attention_mask.dtype) + extended_attention_mask = causal_mask * attention_mask[:, None, :] attention_mask_shape = shape_list(extended_attention_mask) extended_attention_mask = tf.reshape( extended_attention_mask, (attention_mask_shape[0], 1, attention_mask_shape[1], attention_mask_shape[2]) ) - if inputs["past_key_values"][0] is not None: + if past_key_values[0] is not None: # attention_mask needs to be sliced to the shape `[batch_size, 1, from_seq_length - cached_seq_length, to_seq_length] extended_attention_mask = extended_attention_mask[:, :, -seq_length:, :] else: extended_attention_mask = tf.reshape( - inputs["attention_mask"], (attention_mask_shape[0], 1, 1, attention_mask_shape[1]) + attention_mask, (attention_mask_shape[0], 1, 1, attention_mask_shape[1]) ) # Since attention_mask is 1.0 for positions we want to attend and 0.0 for @@ -752,18 +734,18 @@ def call( extended_attention_mask = tf.multiply(tf.subtract(one_cst, extended_attention_mask), ten_thousand_cst) # Copied from `modeling_tf_t5.py` with -1e9 -> -10000 - if self.is_decoder and inputs["encoder_attention_mask"] is not None: + if self.is_decoder and encoder_attention_mask is not None: # If a 2D ou 3D attention mask is provided for the cross-attention # we need to make broadcastable to [batch_size, num_heads, mask_seq_length, mask_seq_length] # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] - inputs["encoder_attention_mask"] = tf.cast( - inputs["encoder_attention_mask"], dtype=extended_attention_mask.dtype + encoder_attention_mask = tf.cast( + encoder_attention_mask, dtype=extended_attention_mask.dtype ) - num_dims_encoder_attention_mask = len(shape_list(inputs["encoder_attention_mask"])) + num_dims_encoder_attention_mask = len(shape_list(encoder_attention_mask)) if num_dims_encoder_attention_mask == 3: - encoder_extended_attention_mask = inputs["encoder_attention_mask"][:, None, :, :] + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] if num_dims_encoder_attention_mask == 2: - encoder_extended_attention_mask = inputs["encoder_attention_mask"][:, None, None, :] + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 @@ -779,28 +761,28 @@ def call( # attention_probs has shape bsz x n_heads x N x N # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] - if inputs["head_mask"] is not None: + if head_mask is not None: raise NotImplementedError else: - inputs["head_mask"] = [None] * self.config.num_hidden_layers + head_mask = [None] * self.config.num_hidden_layers encoder_outputs = self.encoder( hidden_states=embedding_output, attention_mask=extended_attention_mask, - head_mask=inputs["head_mask"], - encoder_hidden_states=inputs["encoder_hidden_states"], + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_extended_attention_mask, - past_key_values=inputs["past_key_values"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, ) sequence_output = encoder_outputs[0] - if not inputs["return_dict"]: + if not return_dict: return ( sequence_output, ) + encoder_outputs[1:] @@ -943,6 +925,7 @@ def __init__(self, config: {{cookiecutter.camelcase_modelname}}Config, *inputs, self.{{cookiecutter.lowercase_modelname}} = TF{{cookiecutter.camelcase_modelname}}MainLayer(config, name="{{cookiecutter.lowercase_modelname}}") + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -988,9 +971,7 @@ def call( If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). Set to `False` during training, `True` during generation """ - inputs = input_processing( - func=self.call, - config=self.config, + outputs = self.{{cookiecutter.lowercase_modelname}}( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1005,23 +986,6 @@ def call( output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - outputs = self.{{cookiecutter.lowercase_modelname}}( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - encoder_hidden_states=inputs["encoder_hidden_states"], - encoder_attention_mask=inputs["encoder_attention_mask"], - past_key_values=inputs["past_key_values"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) return outputs @@ -1064,6 +1028,7 @@ def __init__(self, config: {{cookiecutter.camelcase_modelname}}Config, *inputs, def get_lm_head(self) -> tf.keras.layers.Layer: return self.mlm.predictions + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1091,9 +1056,7 @@ def call( Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` """ - inputs = input_processing( - func=self.call, - config=self.config, + outputs = self.{{cookiecutter.lowercase_modelname}}( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1103,29 +1066,15 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - outputs = self.{{cookiecutter.lowercase_modelname}}( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) sequence_output = outputs[0] - prediction_scores = self.mlm(sequence_output=sequence_output, training=inputs["training"]) + prediction_scores = self.mlm(sequence_output=sequence_output, training=training) loss = ( - None if inputs["labels"] is None else self.hf_compute_loss(labels=inputs["labels"], logits=prediction_scores) + None if labels is None else self.hf_compute_loss(labels=labels, logits=prediction_scores) ) - if not inputs["return_dict"]: + if not return_dict: output = (prediction_scores,) + outputs[2:] return ((loss,) + output) if loss is not None else output @@ -1173,6 +1122,7 @@ def prepare_inputs_for_generation(self, inputs, past=None, attention_mask=None, "use_cache": model_kwargs["use_cache"], } + @unpack_inputs @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, @@ -1220,9 +1170,7 @@ def call( labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`. """ - inputs = input_processing( - func=self.call, - config=self.config, + outputs = self.{{cookiecutter.lowercase_modelname}}( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1236,37 +1184,19 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - outputs = self.{{cookiecutter.lowercase_modelname}}( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - encoder_hidden_states=inputs["encoder_hidden_states"], - encoder_attention_mask=inputs["encoder_attention_mask"], - past_key_values=inputs["past_key_values"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) sequence_output = outputs[0] - logits = self.mlm(sequence_output=sequence_output, training=inputs["training"]) + logits = self.mlm(sequence_output=sequence_output, training=training) loss = None - if inputs["labels"] is not None: + if labels is not None: # shift labels to the left and cut last logit token shifted_logits = logits[:, :-1] - labels = inputs["labels"][:, 1:] + labels = labels[:, 1:] loss = self.hf_compute_loss(labels=labels, logits=shifted_logits) - if not inputs["return_dict"]: + if not return_dict: output = (logits,) + outputs[2:] return ((loss,) + output) if loss is not None else output @@ -1338,6 +1268,7 @@ def __init__(self, config: {{cookiecutter.camelcase_modelname}}Config, *inputs, self.{{cookiecutter.lowercase_modelname}} = TF{{cookiecutter.camelcase_modelname}}MainLayer(config, name="{{cookiecutter.lowercase_modelname}}") self.classifier = TF{{cookiecutter.camelcase_modelname}}ClassificationHead(config, name="classifier") + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1365,9 +1296,7 @@ def call( Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). """ - inputs = input_processing( - func=self.call, - config=self.config, + outputs = self.{{cookiecutter.lowercase_modelname}}( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1377,26 +1306,12 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, ) - outputs = self.{{cookiecutter.lowercase_modelname}}( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], - ) - logits = self.classifier(hidden_states=outputs[0], training=inputs["training"]) - loss = None if inputs["labels"] is None else self.hf_compute_loss(labels=inputs["labels"], logits=logits) - - if not inputs["return_dict"]: + logits = self.classifier(hidden_states=outputs[0], training=training) + loss = None if labels is None else self.hf_compute_loss(labels=labels, logits=logits) + + if not return_dict: output = (logits,) + outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1443,6 +1358,7 @@ def dummy_inputs(self) -> Dict[str, tf.Tensor]: """ return {"input_ids": tf.constant(MULTIPLE_CHOICE_DUMMY_INPUTS)} + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1470,53 +1386,37 @@ def call( Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - labels=labels, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None: - num_choices = shape_list(inputs["input_ids"])[1] - seq_length = shape_list(inputs["input_ids"])[2] + if input_ids is not None: + num_choices = shape_list(input_ids)[1] + seq_length = shape_list(input_ids)[2] else: - num_choices = shape_list(inputs["inputs_embeds"])[1] - seq_length = shape_list(inputs["inputs_embeds"])[2] + num_choices = shape_list(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[2] flat_input_ids = ( - tf.reshape(tensor=inputs["input_ids"], shape=(-1, seq_length)) if inputs["input_ids"] is not None else None + tf.reshape(tensor=input_ids, shape=(-1, seq_length)) if input_ids is not None else None ) flat_attention_mask = ( - tf.reshape(tensor=inputs["attention_mask"], shape=(-1, seq_length)) - if inputs["attention_mask"] is not None + tf.reshape(tensor=attention_mask, shape=(-1, seq_length)) + if attention_mask is not None else None ) flat_token_type_ids = ( - tf.reshape(tensor=inputs["token_type_ids"], shape=(-1, seq_length)) - if inputs["token_type_ids"] is not None + tf.reshape(tensor=token_type_ids, shape=(-1, seq_length)) + if token_type_ids is not None else None ) flat_position_ids = ( - tf.reshape(tensor=inputs["position_ids"], shape=(-1, seq_length)) - if inputs["position_ids"] is not None + tf.reshape(tensor=position_ids, shape=(-1, seq_length)) + if position_ids is not None else None ) flat_inputs_embeds = ( tf.reshape( - tensor=inputs["inputs_embeds"], shape=(-1, seq_length, shape_list(inputs["inputs_embeds"])[3]) + tensor=inputs_embeds, shape=(-1, seq_length, shape_list(inputs_embeds)[3]) ) - if inputs["inputs_embeds"] is not None + if inputs_embeds is not None else None ) outputs = self.{{cookiecutter.lowercase_modelname}}( @@ -1524,19 +1424,19 @@ def call( attention_mask=flat_attention_mask, token_type_ids=flat_token_type_ids, position_ids=flat_position_ids, - head_mask=inputs["head_mask"], + head_mask=head_mask, inputs_embeds=flat_inputs_embeds, - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, ) - logits = self.sequence_summary(inputs=outputs[0], training=inputs["training"]) + logits = self.sequence_summary(inputs=outputs[0], training=training) logits = self.classifier(inputs=logits) reshaped_logits = tf.reshape(tensor=logits, shape=(-1, num_choices)) - loss = None if inputs["labels"] is None else self.hf_compute_loss(labels=inputs["labels"], logits=reshaped_logits) + loss = None if labels is None else self.hf_compute_loss(labels=labels, logits=reshaped_logits) - if not inputs["return_dict"]: + if not return_dict: output = (reshaped_logits,) + outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1585,6 +1485,7 @@ def __init__(self, config: {{cookiecutter.camelcase_modelname}}Config, *inputs, units=config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="classifier" ) + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1611,9 +1512,8 @@ def call( labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. """ - inputs = input_processing( - func=self.call, - config=self.config, + + outputs = self.{{cookiecutter.lowercase_modelname}}( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1623,28 +1523,14 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, training=training, - kwargs_call=kwargs, - ) - outputs = self.{{cookiecutter.lowercase_modelname}}( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) sequence_output = outputs[0] - sequence_output = self.dropout(inputs=sequence_output, training=inputs["training"]) + sequence_output = self.dropout(inputs=sequence_output, training=training) logits = self.classifier(inputs=sequence_output) - loss = None if inputs["labels"] is None else self.hf_compute_loss(labels=inputs["labels"], logits=logits) + loss = None if labels is None else self.hf_compute_loss(labels=labels, logits=logits) - if not inputs["return_dict"]: + if not return_dict: output = (logits,) + outputs[1:] return ((loss,) + output) if loss is not None else output @@ -1680,6 +1566,7 @@ def __init__(self, config: {{cookiecutter.camelcase_modelname}}Config, *inputs, units=config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="qa_outputs" ) + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -1713,9 +1600,7 @@ def call( Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. """ - inputs = input_processing( - func=self.call, - config=self.config, + outputs = self.{{cookiecutter.lowercase_modelname}}( input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, @@ -1725,22 +1610,7 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - start_positions=start_positions, - end_positions=end_positions, training=training, - kwargs_call=kwargs, - ) - outputs = self.{{cookiecutter.lowercase_modelname}}( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - token_type_ids=inputs["token_type_ids"], - position_ids=inputs["position_ids"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) sequence_output = outputs[0] logits = self.qa_outputs(inputs=sequence_output) @@ -1749,12 +1619,12 @@ def call( end_logits = tf.squeeze(input=end_logits, axis=-1) loss = None - if inputs["start_positions"] is not None and inputs["end_positions"] is not None: - labels = {"start_position": inputs["start_positions"]} - labels["end_position"] = inputs["end_positions"] + if start_positions is not None and end_positions is not None: + labels = {"start_position": start_positions} + labels["end_position"] = end_positions loss = self.hf_compute_loss(labels=labels, logits=(start_logits, end_logits)) - if not inputs["return_dict"]: + if not return_dict: output = (start_logits, end_logits) + outputs[2:] return ((loss,) + output) if loss is not None else output @@ -1801,8 +1671,8 @@ def serving_output(self, output: TFQuestionAnsweringModelOutput) -> TFQuestionAn TFPreTrainedModel, TFSharedEmbeddings, TFWrappedEmbeddings, - input_processing, keras_serializable, + unpack_inputs, ); from ...tf_utils import (shape_list, ) from ...utils import logging @@ -2381,6 +2251,7 @@ def get_embed_tokens(self): def set_embed_tokens(self, embed_tokens): self.embed_tokens = embed_tokens + @unpack_inputs def call( self, input_ids=None, @@ -2435,78 +2306,65 @@ def call( Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") - if inputs["inputs_embeds"] is None: - inputs["inputs_embeds"] = self.embed_tokens(inputs["input_ids"]) * self.embed_scale + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale embed_pos = self.embed_positions(input_shape) - hidden_states = inputs["inputs_embeds"] + embed_pos + hidden_states = inputs_embeds + embed_pos hidden_states = self.layernorm_embedding(hidden_states) - hidden_states = self.dropout(hidden_states, training=inputs["training"]) + hidden_states = self.dropout(hidden_states, training=training) # check attention mask and invert - if inputs["attention_mask"] is not None: + if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - inputs["attention_mask"] = _expand_mask(inputs["attention_mask"]) + attention_mask = _expand_mask(attention_mask) - encoder_states = () if inputs["output_hidden_states"] else None - all_attentions = () if inputs["output_attentions"] else None + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None # check if head_mask has a correct number of layers specified if desired # The tf.debugging asserts are not compliant with XLA then they # have to be disabled in other modes than eager. - if inputs["head_mask"] is not None and tf.executing_eagerly(): + if head_mask is not None and tf.executing_eagerly(): tf.debugging.assert_equal( - shape_list(inputs["head_mask"])[0], + shape_list(head_mask)[0], len(self.layers), - message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(inputs['head_mask'])[0]}.", + message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(head_mask)[0]}.", ) # encoder layers for idx, encoder_layer in enumerate(self.layers): - if inputs["output_hidden_states"]: + if output_hidden_states: encoder_states = encoder_states + (hidden_states,) # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) dropout_probability = random.uniform(0, 1) - if inputs["training"] and (dropout_probability < self.layerdrop): # skip the layer + if training and (dropout_probability < self.layerdrop): # skip the layer continue hidden_states, attn = encoder_layer( hidden_states, - inputs["attention_mask"], - inputs["head_mask"][idx] if inputs["head_mask"] is not None else None, + attention_mask, + head_mask[idx] if head_mask is not None else None, ) - if inputs["output_attentions"]: + if output_attentions: all_attentions += (attn,) - if inputs["output_hidden_states"]: + if output_hidden_states: encoder_states = encoder_states + (hidden_states,) - if not inputs["return_dict"]: + if not return_dict: return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) return TFBaseModelOutput( last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions @@ -2547,6 +2405,7 @@ def get_embed_tokens(self): def set_embed_tokens(self, embed_tokens): self.embed_tokens = embed_tokens + @unpack_inputs def call( self, input_ids=None, @@ -2632,111 +2491,93 @@ def call( Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). """ - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - head_mask=head_mask, - cross_attn_head_mask=cross_attn_head_mask, - inputs_embeds=inputs_embeds, - past_key_values=past_key_values, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - kwargs_call=kwargs, - ) - if inputs["input_ids"] is not None and inputs["inputs_embeds"] is not None: + if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") - elif inputs["input_ids"] is not None: - input_shape = shape_list(inputs["input_ids"]) - elif inputs["inputs_embeds"] is not None: - input_shape = shape_list(inputs["inputs_embeds"])[:-1] + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") past_key_values_length = ( - shape_list(inputs["past_key_values"][0][0])[2] if inputs["past_key_values"] is not None else 0 + shape_list(past_key_values[0][0])[2] if past_key_values is not None else 0 ) # embed positions positions = self.embed_positions(input_shape, past_key_values_length) - if inputs["inputs_embeds"] is None: - inputs["inputs_embeds"] = self.embed_tokens(inputs["input_ids"]) + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) - hidden_states = inputs["inputs_embeds"] + hidden_states = inputs_embeds - inputs["attention_mask"], combined_attention_mask = self.compute_combined_attns_mask( - inputs, input_shape, past_key_values_length + attention_mask, combined_attention_mask = self.compute_combined_attns_mask( + input_ids, attention_mask, input_shape, past_key_values_length ) - if inputs["encoder_hidden_states"] is not None and inputs["encoder_attention_mask"] is not None: + if encoder_hidden_states is not None and encoder_attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - inputs["encoder_attention_mask"] = _expand_mask(inputs["encoder_attention_mask"], tgt_len=input_shape[-1]) + encoder_attention_mask = _expand_mask(encoder_attention_mask, tgt_len=input_shape[-1]) hidden_states = self.layernorm_embedding(hidden_states + positions) - hidden_states = self.dropout(hidden_states, training=inputs["training"]) + hidden_states = self.dropout(hidden_states, training=training) # decoder layers - all_hidden_states = () if inputs["output_hidden_states"] else None - all_self_attns = () if inputs["output_attentions"] else None - all_cross_attns = () if (inputs["output_attentions"] and inputs["encoder_hidden_states"] is not None) else None - present_key_values = () if inputs["use_cache"] else None + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attns = () if (output_attentions and encoder_hidden_states is not None) else None + present_key_values = () if use_cache else None # check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired # The tf.debugging asserts are not compliant with XLA then they # have to be disabled in other modes than eager. - for attn_mask in ["head_mask", "cross_attn_head_mask"]: - if inputs[attn_mask] is not None and tf.executing_eagerly(): + for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]: + if attn_mask is not None and tf.executing_eagerly(): tf.debugging.assert_equal( - shape_list(inputs[attn_mask])[0], + shape_list(attn_mask)[0], len(self.layers), - message=f"The {attn_mask} should be specified for {len(self.layers)} layers, but it is for {shape_list(inputs[attn_mask])[0]}.", + message=f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", ) for idx, decoder_layer in enumerate(self.layers): # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) - if inputs["output_hidden_states"]: + if output_hidden_states: all_hidden_states += (hidden_states,) dropout_probability = random.uniform(0, 1) - if inputs["training"] and (dropout_probability < self.layerdrop): + if training and (dropout_probability < self.layerdrop): continue - past_key_value = inputs["past_key_values"][idx] if inputs["past_key_values"] is not None else None + past_key_value = past_key_values[idx] if past_key_values is not None else None hidden_states, layer_self_attn, layer_cross_attn, present_key_value = decoder_layer( hidden_states, attention_mask=combined_attention_mask, - encoder_hidden_states=inputs["encoder_hidden_states"], - encoder_attention_mask=inputs["encoder_attention_mask"], - layer_head_mask=inputs["head_mask"][idx] if inputs["head_mask"] is not None else None, - cross_attn_layer_head_mask=inputs["cross_attn_head_mask"][idx] - if inputs["cross_attn_head_mask"] is not None + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + layer_head_mask=head_mask[idx] if head_mask is not None else None, + cross_attn_layer_head_mask=cross_attn_head_mask[idx] + if cross_attn_head_mask is not None else None, past_key_value=past_key_value, ) - if inputs["use_cache"]: + if use_cache: present_key_values += (present_key_value,) - if inputs["output_attentions"]: + if output_attentions: all_self_attns += (layer_self_attn,) - if inputs["encoder_hidden_states"] is not None: + if encoder_hidden_states is not None: all_cross_attns += (layer_cross_attn,) - if inputs["output_hidden_states"]: + if output_hidden_states: all_hidden_states += (hidden_states,) - if not inputs["return_dict"]: + if not return_dict: return hidden_states, present_key_values, all_hidden_states, all_self_attns, all_cross_attns else: return TFBaseModelOutputWithPastAndCrossAttentions( @@ -2748,7 +2589,7 @@ def call( ) @tf.function - def compute_combined_attns_mask(self, inputs, input_shape, past_key_values_length): + def compute_combined_attns_mask(self, input_ids, attention_mask, input_shape, past_key_values_length): # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] combined_attention_mask = None if input_shape[-1] > 1: @@ -2758,9 +2599,9 @@ def compute_combined_attns_mask(self, inputs, input_shape, past_key_values_lengt tf.ones((input_shape[0], input_shape[1] + past_key_values_length)), tgt_len=input_shape[-1] ) - if inputs["attention_mask"] is None and inputs["input_ids"] is not None and input_shape[-1] > 1: + if attention_mask is None and input_ids is not None and input_shape[-1] > 1: attention_mask = tf.cast( - tf.math.not_equal(inputs["input_ids"], self.config.pad_token_id), inputs["input_ids"].dtype + tf.math.not_equal(input_ids, self.config.pad_token_id), input_ids.dtype ) attention_mask = tf.concat( [ @@ -2810,6 +2651,7 @@ def set_input_embeddings(self, new_embeddings): self.encoder.set_embed_tokens(embed_tokens) self.decoder.set_embed_tokens(embed_tokens) + @unpack_inputs def call( self, input_ids=None, @@ -2830,71 +2672,50 @@ def call( training=False, **kwargs ): - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, + + if decoder_input_ids is None and decoder_inputs_embeds is None: + use_cache = False + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a TFBaseModelOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, TFBaseModelOutput): + encoder_outputs = TFBaseModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + ) + # If the user passed a TFBaseModelOutput for encoder_outputs, we wrap it in a tuple when return_dict=False + elif not return_dict and not isinstance(encoder_outputs, tuple): + encoder_outputs = encoder_outputs.to_tuple() + + decoder_outputs = self.decoder( + decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_outputs[0], + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, cross_attn_head_mask=cross_attn_head_mask, - encoder_outputs=encoder_outputs, past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, + inputs_embeds=decoder_inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - - if inputs["decoder_input_ids"] is None and inputs["decoder_inputs_embeds"] is None: - inputs["use_cache"] = False - - if inputs["encoder_outputs"] is None: - inputs["encoder_outputs"] = self.encoder( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - head_mask=inputs["head_mask"], - inputs_embeds=inputs["inputs_embeds"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], - ) - # If the user passed a tuple for encoder_outputs, we wrap it in a TFBaseModelOutput when return_dict=True - elif inputs["return_dict"] and not isinstance(inputs["encoder_outputs"], TFBaseModelOutput): - inputs["encoder_outputs"] = TFBaseModelOutput( - last_hidden_state=inputs["encoder_outputs"][0], - hidden_states=inputs["encoder_outputs"][1] if len(inputs["encoder_outputs"]) > 1 else None, - attentions=inputs["encoder_outputs"][2] if len(inputs["encoder_outputs"]) > 2 else None, - ) - # If the user passed a TFBaseModelOutput for encoder_outputs, we wrap it in a tuple when return_dict=False - elif not inputs["return_dict"] and not isinstance(inputs["encoder_outputs"], tuple): - inputs["encoder_outputs"] = inputs["encoder_outputs"].to_tuple() + ) - decoder_outputs = self.decoder( - inputs["decoder_input_ids"], - attention_mask=inputs["decoder_attention_mask"], - encoder_hidden_states=inputs["encoder_outputs"][0], - encoder_attention_mask=inputs["attention_mask"], - head_mask=inputs["decoder_head_mask"], - cross_attn_head_mask=inputs["cross_attn_head_mask"], - past_key_values=inputs["past_key_values"], - inputs_embeds=inputs["decoder_inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], - ) - - if not inputs["return_dict"]: - return decoder_outputs + inputs["encoder_outputs"] + if not return_dict: + return decoder_outputs + encoder_outputs return TFSeq2SeqModelOutput( last_hidden_state=decoder_outputs.last_hidden_state, @@ -2902,9 +2723,9 @@ def call( decoder_hidden_states=decoder_outputs.hidden_states, decoder_attentions=decoder_outputs.attentions, cross_attentions=decoder_outputs.cross_attentions, - encoder_last_hidden_state=inputs["encoder_outputs"].last_hidden_state, - encoder_hidden_states=inputs["encoder_outputs"].hidden_states, - encoder_attentions=inputs["encoder_outputs"].attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, ) @@ -2924,6 +2745,7 @@ def get_encoder(self): def get_decoder(self): return self.model.decoder + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, @@ -2951,9 +2773,8 @@ def call( training=False, **kwargs ): - inputs = input_processing( - func=self.call, - config=self.config, + + outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, @@ -2970,26 +2791,6 @@ def call( output_hidden_states=output_hidden_states, return_dict=return_dict, training=training, - kwargs_call=kwargs, - ) - - outputs = self.model( - input_ids=inputs["input_ids"], - attention_mask=inputs["attention_mask"], - decoder_input_ids=inputs["decoder_input_ids"], - decoder_attention_mask=inputs["decoder_attention_mask"], - head_mask=inputs["head_mask"], - decoder_head_mask=inputs["decoder_head_mask"], - cross_attn_head_mask=inputs["cross_attn_head_mask"], - encoder_outputs=inputs["encoder_outputs"], - past_key_values=inputs["past_key_values"], - inputs_embeds=inputs["inputs_embeds"], - decoder_inputs_embeds=inputs["decoder_inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"], ) return outputs @@ -3053,6 +2854,7 @@ def get_output_embeddings(self): def set_output_embeddings(self, value): self.set_input_embeddings(value) + @unpack_inputs @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) def call( @@ -3093,17 +2895,23 @@ def call( >>> probs = tf.nn.softmax(logits[0]) >>> # probs[5] is associated with the mask token ```""" - inputs = input_processing( - func=self.call, - config=self.config, - input_ids=input_ids, + + if labels is not None: + use_cache = False + if decoder_input_ids is None: + decoder_input_ids = shift_tokens_right( + labels, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + outputs = self.model( + input_ids, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, + encoder_outputs=encoder_outputs, decoder_attention_mask=decoder_attention_mask, head_mask=head_mask, decoder_head_mask=decoder_head_mask, cross_attn_head_mask=cross_attn_head_mask, - encoder_outputs=encoder_outputs, past_key_values=past_key_values, inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, @@ -3111,41 +2919,13 @@ def call( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - labels=labels, - training=training, - kwargs_call=kwargs, - ) - - if inputs["labels"] is not None: - inputs["use_cache"] = False - if inputs["decoder_input_ids"] is None: - inputs["decoder_input_ids"] = shift_tokens_right( - inputs["labels"], self.config.pad_token_id, self.config.decoder_start_token_id - ) - - outputs = self.model( - inputs["input_ids"], - attention_mask=inputs["attention_mask"], - decoder_input_ids=inputs["decoder_input_ids"], - encoder_outputs=inputs["encoder_outputs"], - decoder_attention_mask=inputs["decoder_attention_mask"], - head_mask=inputs["head_mask"], - decoder_head_mask=inputs["decoder_head_mask"], - cross_attn_head_mask=inputs["cross_attn_head_mask"], - past_key_values=inputs["past_key_values"], - inputs_embeds=inputs["inputs_embeds"], - decoder_inputs_embeds=inputs["decoder_inputs_embeds"], - use_cache=inputs["use_cache"], - output_attentions=inputs["output_attentions"], - output_hidden_states=inputs["output_hidden_states"], - return_dict=inputs["return_dict"], - training=inputs["training"] + training=training ) lm_logits = self.model.shared(outputs[0], mode="linear") lm_logits = lm_logits + self.final_logits_bias - masked_lm_loss = None if inputs["labels"] is None else self.hf_compute_loss(inputs["labels"], lm_logits) + masked_lm_loss = None if labels is None else self.hf_compute_loss(labels, lm_logits) - if not inputs["return_dict"]: + if not return_dict: output = (lm_logits,) + outputs[1:] return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output return TFSeq2SeqLMOutput( From c4deb7b3ae64f6a4bd0e86cdbd3985de4d24b46e Mon Sep 17 00:00:00 2001 From: Francesco Saverio Zuppichini Date: Wed, 30 Mar 2022 18:46:51 +0200 Subject: [PATCH 27/54] Feature Extractor accepts `segmentation_maps` (#15964) * feature extractor accepts * resolved conversations * added examples in test for ADE20K * num_classes -> num_labels * Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * resolving conversations * resolving conversations * removed ADE * CI * minor changes in conversion script * reduce_labels in feature extractor * minor changes * correct preprocess for instace segmentation maps * minor changes * minor changes * CI * debugging * better padding * going to update labels inside the model * going to update labels inside the model * minor changes * tests * removed changes in feature_extractor_utils * conversation * conversation * example in feature extractor * more docstring in modeling * test * make style * doc Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- ..._original_pytorch_checkpoint_to_pytorch.py | 9 +- .../feature_extraction_maskformer.py | 243 ++++++++++++------ .../models/maskformer/modeling_maskformer.py | 164 +++++++----- .../test_feature_extraction_maskformer.py | 90 ++++--- tests/maskformer/test_modeling_maskformer.py | 13 +- 5 files changed, 327 insertions(+), 192 deletions(-) diff --git a/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py index d4041ed59aad..045d2bc0f515 100644 --- a/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py @@ -169,12 +169,15 @@ class OriginalMaskFormerConfigToFeatureExtractorConverter: def __call__(self, original_config: object) -> MaskFormerFeatureExtractor: model = original_config.MODEL model_input = original_config.INPUT + dataset_catalog = MetadataCatalog.get(original_config.DATASETS.TEST[0]) return MaskFormerFeatureExtractor( image_mean=(torch.tensor(model.PIXEL_MEAN) / 255).tolist(), image_std=(torch.tensor(model.PIXEL_STD) / 255).tolist(), size=model_input.MIN_SIZE_TEST, max_size=model_input.MAX_SIZE_TEST, + num_labels=model.SEM_SEG_HEAD.NUM_CLASSES, + ignore_index=dataset_catalog.ignore_label, size_divisibility=32, # 32 is required by swin ) @@ -552,7 +555,7 @@ def using_dirs(checkpoints_dir: Path, config_dir: Path) -> Iterator[Tuple[object yield config, checkpoint -def test(original_model, our_model: MaskFormerForInstanceSegmentation): +def test(original_model, our_model: MaskFormerForInstanceSegmentation, feature_extractor: MaskFormerFeatureExtractor): with torch.no_grad(): original_model = original_model.eval() @@ -600,8 +603,6 @@ def test(original_model, our_model: MaskFormerForInstanceSegmentation): our_model_out: MaskFormerForInstanceSegmentationOutput = our_model(x) - feature_extractor = MaskFormerFeatureExtractor() - our_segmentation = feature_extractor.post_process_segmentation(our_model_out, target_size=(384, 384)) assert torch.allclose( @@ -707,7 +708,7 @@ def get_name(checkpoint_file: Path): mask_former_for_instance_segmentation ) - test(original_model, mask_former_for_instance_segmentation) + test(original_model, mask_former_for_instance_segmentation, feature_extractor) model_name = get_name(checkpoint_file) logger.info(f"🪄 Saving {model_name}") diff --git a/src/transformers/models/maskformer/feature_extraction_maskformer.py b/src/transformers/models/maskformer/feature_extraction_maskformer.py index bd8adc04d0b7..5e466f2ddb07 100644 --- a/src/transformers/models/maskformer/feature_extraction_maskformer.py +++ b/src/transformers/models/maskformer/feature_extraction_maskformer.py @@ -54,6 +54,10 @@ class MaskFormerFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionM max_size (`int`, *optional*, defaults to 1333): The largest size an image dimension can have (otherwise it's capped). Only has an effect if `do_resize` is set to `True`. + resample (`int`, *optional*, defaults to `PIL.Image.BILINEAR`): + An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`, + `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect + if `do_resize` is set to `True`. size_divisibility (`int`, *optional*, defaults to 32): Some backbones need images divisible by a certain number. If not passed, it defaults to the value used in Swin Transformer. @@ -64,8 +68,12 @@ class MaskFormerFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionM image_std (`int`, *optional*, defaults to `[0.229, 0.224, 0.225]`): The sequence of standard deviations for each channel, to be used when normalizing images. Defaults to the ImageNet std. - ignore_index (`int`, *optional*, default to 255): - Value of the index (label) to ignore. + ignore_index (`int`, *optional*): + Value of the index (label) to be removed from the segmentation maps. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is + used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). The + background label will be replaced by `ignore_index`. """ @@ -76,24 +84,28 @@ def __init__( do_resize=True, size=800, max_size=1333, + resample=Image.BILINEAR, size_divisibility=32, do_normalize=True, image_mean=None, image_std=None, - ignore_index=255, + ignore_index=None, + reduce_labels=False, **kwargs ): super().__init__(**kwargs) self.do_resize = do_resize self.size = size self.max_size = max_size + self.resample = resample self.size_divisibility = size_divisibility - self.ignore_index = ignore_index self.do_normalize = do_normalize self.image_mean = image_mean if image_mean is not None else [0.485, 0.456, 0.406] # ImageNet mean self.image_std = image_std if image_std is not None else [0.229, 0.224, 0.225] # ImageNet std + self.ignore_index = ignore_index + self.reduce_labels = reduce_labels - def _resize(self, image, size, target=None, max_size=None): + def _resize_with_size_divisibility(self, image, size, target=None, max_size=None): """ Resize the image to the given size. Size can be min_size (scalar) or (width, height) tuple. If size is an int, smaller edge of the image will be matched to this number. @@ -138,30 +150,19 @@ def get_size(image_size, size, max_size=None): width = int(np.ceil(width / self.size_divisibility)) * self.size_divisibility size = (width, height) - rescaled_image = self.resize(image, size=size) - - has_target = target is not None + image = self.resize(image, size=size, resample=self.resample) - if has_target: - target = target.copy() - # store original_size - target["original_size"] = image.size - if "masks" in target: - masks = torch.from_numpy(target["masks"])[:, None].float() - # use PyTorch as current workaround - # TODO replace by self.resize - interpolated_masks = ( - nn.functional.interpolate(masks, size=(height, width), mode="nearest")[:, 0] > 0.5 - ).float() - target["masks"] = interpolated_masks.numpy() + if target is not None: + target = self.resize(target, size=size, resample=Image.NEAREST) - return rescaled_image, target + return image, target def __call__( self, images: ImageInput, - annotations: Union[List[Dict], List[List[Dict]]] = None, + segmentation_maps: ImageInput = None, pad_and_return_pixel_mask: Optional[bool] = True, + instance_id_to_semantic_id: Optional[Dict[int, int]] = None, return_tensors: Optional[Union[str, TensorType]] = None, **kwargs, ) -> BatchFeature: @@ -170,6 +171,12 @@ def __call__( padded up to the largest image in a batch, and a pixel mask is created that indicates which pixels are real/which are padding. + MaskFormer addresses semantic segmentation with a mask classification paradigm, thus input segmentation maps + will be converted to lists of binary masks and their respective labels. Let's see an example, assuming + `segmentation_maps = [[2,6,7,9]]`, the output will contain `mask_labels = + [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]` (four binary masks) and `class_labels = [2,6,7,9]`, the labels for + each mask. + NumPy arrays and PyTorch tensors are converted to PIL images when resizing, so the most efficient is to pass @@ -183,10 +190,8 @@ def __call__( tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a number of channels, H and W are image height and width. - annotations (`Dict`, `List[Dict]`, *optional*): - The corresponding annotations as dictionary of numpy arrays with the following keys: - - **masks** (`np.ndarray`) The target mask of shape `(num_classes, height, width)`. - - **labels** (`np.ndarray`) The target labels of shape `(num_classes)`. + segmentation_maps (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*): + Optionally, the corresponding semantic segmentation maps with the pixel-wise annotations. pad_and_return_pixel_mask (`bool`, *optional*, defaults to `True`): Whether or not to pad images up to the largest image in a batch and create a pixel mask. @@ -196,7 +201,12 @@ def __call__( - 1 for pixels that are real (i.e. **not masked**), - 0 for pixels that are padding (i.e. **masked**). - return_tensors (`str` or [`~utils.TensorType`], *optional*): + instance_id_to_semantic_id (`Dict[int, int]`, *optional*): + If passed, we treat `segmentation_maps` as an instance segmentation map where each pixel represents an + instance id. To convert it to a binary mask of shape (`batch, num_labels, height, width`) we need a + dictionary mapping instance ids to label ids to create a semantic segmentation map. + + return_tensors (`str` or [`~file_utils.TensorType`], *optional*): If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch `torch.Tensor` objects. @@ -206,15 +216,16 @@ def __call__( - **pixel_values** -- Pixel values to be fed to a model. - **pixel_mask** -- Pixel mask to be fed to a model (when `pad_and_return_pixel_mask=True` or if *"pixel_mask"* is in `self.model_input_names`). - - **mask_labels** -- Optional mask labels of shape `(batch_size, num_classes, height, width) to be fed to a - model (when `annotations` are provided). - - **class_labels** -- Optional class labels of shape `(batch_size, num_classes) to be fed to a model (when - `annotations` are provided). + - **mask_labels** -- Optional list of mask labels of shape `(labels, height, width)` to be fed to a model + (when `annotations` are provided). + - **class_labels** -- Optional list of class labels of shape `(labels)` to be fed to a model (when + `annotations` are provided). They identify the labels of `mask_labels`, e.g. the label of + `mask_labels[i][j]` if `class_labels[i][j]`. """ # Input type checking for clearer error valid_images = False - valid_annotations = False + valid_segmentation_maps = False # Check that images has a valid type if isinstance(images, (Image.Image, np.ndarray)) or is_torch_tensor(images): @@ -228,6 +239,23 @@ def __call__( "Images must of type `PIL.Image.Image`, `np.ndarray` or `torch.Tensor` (single example), " "`List[PIL.Image.Image]`, `List[np.ndarray]` or `List[torch.Tensor]` (batch of examples)." ) + # Check that segmentation maps has a valid type + if segmentation_maps is not None: + if isinstance(segmentation_maps, (Image.Image, np.ndarray)) or is_torch_tensor(segmentation_maps): + valid_segmentation_maps = True + elif isinstance(segmentation_maps, (list, tuple)): + if ( + len(segmentation_maps) == 0 + or isinstance(segmentation_maps[0], (Image.Image, np.ndarray)) + or is_torch_tensor(segmentation_maps[0]) + ): + valid_segmentation_maps = True + + if not valid_segmentation_maps: + raise ValueError( + "Segmentation maps must of type `PIL.Image.Image`, `np.ndarray` or `torch.Tensor` (single example)," + "`List[PIL.Image.Image]`, `List[np.ndarray]` or `List[torch.Tensor]` (batch of examples)." + ) is_batched = bool( isinstance(images, (list, tuple)) @@ -236,35 +264,33 @@ def __call__( if not is_batched: images = [images] - if annotations is not None: - annotations = [annotations] - - # Check that annotations has a valid type - if annotations is not None: - valid_annotations = type(annotations) is list and "masks" in annotations[0] and "labels" in annotations[0] - if not valid_annotations: - raise ValueError( - "Annotations must of type `Dict` (single image) or `List[Dict]` (batch of images)." - "The annotations must be numpy arrays in the following format:" - "{ 'masks' : the target mask, with shape [C,H,W], 'labels' : the target labels, with shape [C]}" - ) + if segmentation_maps is not None: + segmentation_maps = [segmentation_maps] # transformations (resizing + normalization) if self.do_resize and self.size is not None: - if annotations is not None: - for idx, (image, target) in enumerate(zip(images, annotations)): - image, target = self._resize(image=image, target=target, size=self.size, max_size=self.max_size) + if segmentation_maps is not None: + for idx, (image, target) in enumerate(zip(images, segmentation_maps)): + image, target = self._resize_with_size_divisibility( + image=image, target=target, size=self.size, max_size=self.max_size + ) images[idx] = image - annotations[idx] = target + segmentation_maps[idx] = target else: for idx, image in enumerate(images): - images[idx] = self._resize(image=image, target=None, size=self.size, max_size=self.max_size)[0] + images[idx] = self._resize_with_size_divisibility( + image=image, target=None, size=self.size, max_size=self.max_size + )[0] if self.do_normalize: images = [self.normalize(image=image, mean=self.image_mean, std=self.image_std) for image in images] # NOTE I will be always forced to pad them them since they have to be stacked in the batch dim encoded_inputs = self.encode_inputs( - images, annotations, pad_and_return_pixel_mask, return_tensors=return_tensors + images, + segmentation_maps, + pad_and_return_pixel_mask, + instance_id_to_semantic_id=instance_id_to_semantic_id, + return_tensors=return_tensors, ) # Convert to TensorType @@ -287,25 +313,57 @@ def _max_by_axis(self, the_list: List[List[int]]) -> List[int]: maxes[index] = max(maxes[index], item) return maxes + def convert_segmentation_map_to_binary_masks( + self, + segmentation_map: "np.ndarray", + instance_id_to_semantic_id: Optional[Dict[int, int]] = None, + ): + if self.reduce_labels: + if self.ignore_index is None: + raise ValueError("`ignore_index` must be set when `reduce_labels` is `True`.") + segmentation_map[segmentation_map == 0] = self.ignore_index + # instances ids start from 1! + segmentation_map -= 1 + segmentation_map[segmentation_map == self.ignore_index - 1] = self.ignore_index + + if instance_id_to_semantic_id is not None: + # segmentation_map will be treated as an instance segmentation map where each pixel is a instance id + # thus it has to be converted to a semantic segmentation map + for instance_id, label_id in instance_id_to_semantic_id.items(): + segmentation_map[segmentation_map == instance_id] = label_id + # get all the labels in the image + labels = np.unique(segmentation_map) + # remove ignore index (if we have one) + if self.ignore_index is not None: + labels = labels[labels != self.ignore_index] + # helping broadcast by making mask [1,W,H] and labels [C, 1, 1] + binary_masks = segmentation_map[None] == labels[:, None, None] + return binary_masks.astype(np.float32), labels.astype(np.int64) + def encode_inputs( self, - pixel_values_list: List["torch.Tensor"], - annotations: Optional[List[Dict]] = None, - pad_and_return_pixel_mask: Optional[bool] = True, + pixel_values_list: List["np.ndarray"], + segmentation_maps: ImageInput = None, + pad_and_return_pixel_mask: bool = True, + instance_id_to_semantic_id: Optional[Dict[int, int]] = None, return_tensors: Optional[Union[str, TensorType]] = None, ): """ Pad images up to the largest image in a batch and create a corresponding `pixel_mask`. + MaskFormer addresses semantic segmentation with a mask classification paradigm, thus input segmentation maps + will be converted to lists of binary masks and their respective labels. Let's see an example, assuming + `segmentation_maps = [[2,6,7,9]]`, the output will contain `mask_labels = + [[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]]` (four binary masks) and `class_labels = [2,6,7,9]`, the labels for + each mask. + Args: pixel_values_list (`List[torch.Tensor]`): List of images (pixel values) to be padded. Each image should be a tensor of shape `(channels, height, width)`. - annotations (`Dict`, `List[Dict]`, *optional*): - The corresponding annotations as dictionary of numpy arrays with the following keys: - - **masks** (`np.ndarray`) The target mask of shape `(num_classes, height, width)`. - - **labels** (`np.ndarray`) The target labels of shape `(num_classes)`. + segmentation_maps (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*): + The corresponding semantic segmentation maps with the pixel-wise annotations. pad_and_return_pixel_mask (`bool`, *optional*, defaults to `True`): Whether or not to pad images up to the largest image in a batch and create a pixel mask. @@ -315,7 +373,12 @@ def encode_inputs( - 1 for pixels that are real (i.e. **not masked**), - 0 for pixels that are padding (i.e. **masked**). - return_tensors (`str` or [`~utils.TensorType`], *optional*): + instance_id_to_semantic_id (`Dict[int, int]`, *optional*): + If passed, we treat `segmentation_maps` as an instance segmentation map where each pixel represents an + instance id. To convert it to a binary mask of shape (`batch, num_labels, height, width`) we need a + dictionary mapping instance ids to label ids to create a semantic segmentation map. + + return_tensors (`str` or [`~file_utils.TensorType`], *optional*): If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch `torch.Tensor` objects. @@ -325,13 +388,29 @@ def encode_inputs( - **pixel_values** -- Pixel values to be fed to a model. - **pixel_mask** -- Pixel mask to be fed to a model (when `pad_and_return_pixel_mask=True` or if *"pixel_mask"* is in `self.model_input_names`). - - **mask_labels** -- Optional mask labels of shape `(batch_size, num_classes, height, width) to be fed to a - model (when `annotations` are provided). - - **class_labels** -- Optional class labels of shape `(batch_size, num_classes) to be fed to a model (when - `annotations` are provided). + - **mask_labels** -- Optional list of mask labels of shape `(labels, height, width)` to be fed to a model + (when `annotations` are provided). + - **class_labels** -- Optional list of class labels of shape `(labels)` to be fed to a model (when + `annotations` are provided). They identify the labels of `mask_labels`, e.g. the label of + `mask_labels[i][j]` if `class_labels[i][j]`. """ max_size = self._max_by_axis([list(image.shape) for image in pixel_values_list]) + + annotations = None + if segmentation_maps is not None: + segmentation_maps = map(np.array, segmentation_maps) + converted_segmentation_maps = [] + for segmentation_map in segmentation_maps: + converted_segmentation_map = self.convert_segmentation_map_to_binary_masks( + segmentation_map, instance_id_to_semantic_id + ) + converted_segmentation_maps.append(converted_segmentation_map) + + annotations = [] + for mask, classes in converted_segmentation_maps: + annotations.append({"masks": mask, "classes": classes}) + channels, height, width = max_size pixel_values = [] pixel_mask = [] @@ -339,35 +418,37 @@ def encode_inputs( class_labels = [] for idx, image in enumerate(pixel_values_list): # create padded image - if pad_and_return_pixel_mask: - padded_image = np.zeros((channels, height, width), dtype=np.float32) - padded_image[: image.shape[0], : image.shape[1], : image.shape[2]] = np.copy(image) - image = padded_image + padded_image = np.zeros((channels, height, width), dtype=np.float32) + padded_image[: image.shape[0], : image.shape[1], : image.shape[2]] = np.copy(image) + image = padded_image pixel_values.append(image) # if we have a target, pad it if annotations: annotation = annotations[idx] masks = annotation["masks"] - if pad_and_return_pixel_mask: - padded_masks = np.zeros((masks.shape[0], height, width), dtype=masks.dtype) - padded_masks[:, : masks.shape[1], : masks.shape[2]] = np.copy(masks) - masks = padded_masks - mask_labels.append(masks) - class_labels.append(annotation["labels"]) - if pad_and_return_pixel_mask: - # create pixel mask - mask = np.zeros((height, width), dtype=np.int64) - mask[: image.shape[1], : image.shape[2]] = True - pixel_mask.append(mask) + # pad mask with `ignore_index` + masks = np.pad( + masks, + ((0, 0), (0, height - masks.shape[1]), (0, width - masks.shape[2])), + constant_values=self.ignore_index, + ) + annotation["masks"] = masks + # create pixel mask + mask = np.zeros((height, width), dtype=np.int64) + mask[: image.shape[1], : image.shape[2]] = True + pixel_mask.append(mask) # return as BatchFeature data = {"pixel_values": pixel_values, "pixel_mask": pixel_mask} - + encoded_inputs = BatchFeature(data=data, tensor_type=return_tensors) + # we cannot batch them since they don't share a common class size if annotations: - data["mask_labels"] = mask_labels - data["class_labels"] = class_labels + for label in annotations: + mask_labels.append(torch.from_numpy(label["masks"])) + class_labels.append(torch.from_numpy(label["classes"])) - encoded_inputs = BatchFeature(data=data, tensor_type=return_tensors) + encoded_inputs["mask_labels"] = mask_labels + encoded_inputs["class_labels"] = class_labels return encoded_inputs diff --git a/src/transformers/models/maskformer/modeling_maskformer.py b/src/transformers/models/maskformer/modeling_maskformer.py index 450ba50b59ed..cc793d886575 100644 --- a/src/transformers/models/maskformer/modeling_maskformer.py +++ b/src/transformers/models/maskformer/modeling_maskformer.py @@ -269,7 +269,7 @@ class MaskFormerForInstanceSegmentationOutput(ModelOutput): A tensor of shape `(batch_size, num_queries, height, width)` representing the proposed masks for each query. masks_queries_logits (`torch.FloatTensor`): - A tensor of shape `(batch_size, num_queries, num_classes + 1)` representing the proposed classes for each + A tensor of shape `(batch_size, num_queries, num_labels + 1)` representing the proposed classes for each query. Note the `+ 1` is needed because we incorporate the null class. encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Last hidden states (final feature map) of the last stage of the encoder model (backbone). @@ -424,7 +424,7 @@ def pair_wise_dice_loss(inputs: Tensor, labels: Tensor) -> Tensor: """ inputs = inputs.sigmoid().flatten(1) numerator = 2 * torch.einsum("nc,mc->nm", inputs, labels) - # using broadcasting to get a [NUM_QUERIES, NUM_CLASSES] matrix + # using broadcasting to get a [num_queries, NUM_CLASSES] matrix denominator = inputs.sum(-1)[:, None] + labels.sum(-1)[None, :] loss = 1 - (numerator + 1) / (denominator + 1) return loss @@ -918,7 +918,9 @@ def forward(self, hidden_states, input_dimensions, head_mask=None, output_attent outputs = self_attention_outputs[1:] # add self attentions if we output attention weights attention_windows = attention_output.view(-1, self.window_size, self.window_size, channels) - shifted_windows = window_reverse(attention_windows, self.window_size, height_pad, width_pad) # B H' W' C + shifted_windows = window_reverse( + attention_windows, self.window_size, height_pad, width_pad + ) # B height' width' C # reverse cyclic shift if self.shift_size > 0: @@ -1621,7 +1623,7 @@ def forward(self, masks_queries_logits, class_queries_logits, mask_labels, class Params: masks_queries_logits (`torch.Tensor`): - A tensor` of dim `batch_size, num_queries, num_classes` with the + A tensor` of dim `batch_size, num_queries, num_labels` with the classification logits. class_queries_logits (`torch.Tensor`): A tensor` of dim `batch_size, num_queries, height, width` with the @@ -1644,24 +1646,23 @@ def forward(self, masks_queries_logits, class_queries_logits, mask_labels, class indices: List[Tuple[np.array]] = [] preds_masks = masks_queries_logits - preds_probs = class_queries_logits.softmax(dim=-1) - # downsample all masks in one go -> save memory - mask_labels = nn.functional.interpolate(mask_labels, size=preds_masks.shape[-2:], mode="nearest") + preds_probs = class_queries_logits # iterate through batch size for pred_probs, pred_mask, target_mask, labels in zip(preds_probs, preds_masks, mask_labels, class_labels): + # downsample the target mask, save memory + target_mask = nn.functional.interpolate(target_mask[:, None], size=pred_mask.shape[-2:], mode="nearest") + pred_probs = pred_probs.softmax(-1) # Compute the classification cost. Contrary to the loss, we don't use the NLL, # but approximate it in 1 - proba[target class]. # The 1 is a constant that doesn't change the matching, it can be ommitted. cost_class = -pred_probs[:, labels] # flatten spatial dimension "q h w -> q (h w)" - num_queries, height, width = pred_mask.shape - pred_mask_flat = pred_mask.view(num_queries, height * width) # [num_queries, H*W] + pred_mask_flat = pred_mask.flatten(1) # [num_queries, height*width] # same for target_mask "c h w -> c (h w)" - num_channels, height, width = target_mask.shape - target_mask_flat = target_mask.view(num_channels, height * width) # [num_total_labels, H*W] - # compute the focal loss between each mask pairs -> shape [NUM_QUERIES, CLASSES] + target_mask_flat = target_mask[:, 0].flatten(1) # [num_total_labels, height*width] + # compute the focal loss between each mask pairs -> shape (num_queries, num_labels) cost_mask = pair_wise_sigmoid_focal_loss(pred_mask_flat, target_mask_flat) - # Compute the dice loss betwen each mask pairs -> shape [NUM_QUERIES, CLASSES] + # Compute the dice loss betwen each mask pairs -> shape (num_queries, num_labels) cost_dice = pair_wise_dice_loss(pred_mask_flat, target_mask_flat) # final cost matrix cost_matrix = self.cost_mask * cost_mask + self.cost_class * cost_class + self.cost_dice * cost_dice @@ -1691,7 +1692,7 @@ def __repr__(self): class MaskFormerLoss(nn.Module): def __init__( self, - num_classes: int, + num_labels: int, matcher: MaskFormerHungarianMatcher, weight_dict: Dict[str, float], eos_coef: float, @@ -1702,7 +1703,7 @@ def __init__( matched ground-truth / prediction (supervise class and mask) Args: - num_classes (`int`): + num_labels (`int`): The number of classes. matcher (`MaskFormerHungarianMatcher`): A torch module that computes the assigments between the predictions and labels. @@ -1714,24 +1715,50 @@ def __init__( super().__init__() requires_backends(self, ["scipy"]) - self.num_classes = num_classes + self.num_labels = num_labels self.matcher = matcher self.weight_dict = weight_dict self.eos_coef = eos_coef - empty_weight = torch.ones(self.num_classes + 1) + empty_weight = torch.ones(self.num_labels + 1) empty_weight[-1] = self.eos_coef self.register_buffer("empty_weight", empty_weight) + def _max_by_axis(self, the_list: List[List[int]]) -> List[int]: + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + def _pad_images_to_max_in_batch(self, tensors: List[Tensor]) -> Tuple[Tensor, Tensor]: + # get the maximum size in the batch + max_size = self._max_by_axis([list(tensor.shape) for tensor in tensors]) + batch_size = len(tensors) + # compute finel size + batch_shape = [batch_size] + max_size + b, _, h, w = batch_shape + # get metadata + dtype = tensors[0].dtype + device = tensors[0].device + padded_tensors = torch.zeros(batch_shape, dtype=dtype, device=device) + padding_masks = torch.ones((b, h, w), dtype=torch.bool, device=device) + # pad the tensors to the size of the biggest one + for tensor, padded_tensor, padding_mask in zip(tensors, padded_tensors, padding_masks): + padded_tensor[: tensor.shape[0], : tensor.shape[1], : tensor.shape[2]].copy_(tensor) + padding_mask[: tensor.shape[1], : tensor.shape[2]] = False + + return padded_tensors, padding_masks + def loss_labels( - self, class_queries_logits: Tensor, class_labels: Tensor, indices: Tuple[np.array] + self, class_queries_logits: Tensor, class_labels: List[Tensor], indices: Tuple[np.array] ) -> Dict[str, Tensor]: """Compute the losses related to the labels using cross entropy. Args: class_queries_logits (`torch.Tensor`): - A tensor of shape `batch_size, num_queries, num_classes` - class_labels (`Dict[str, Tensor]`): - A tensor of shape `batch_size, num_classes` + A tensor of shape `batch_size, num_queries, num_labels` + class_labels (`List[torch.Tensor]`): + List of class labels of shape `(labels)`. indices (`Tuple[np.array])`: The indices computed by the Hungarian matcher. @@ -1744,21 +1771,21 @@ def loss_labels( batch_size, num_queries, _ = pred_logits.shape criterion = nn.CrossEntropyLoss(weight=self.empty_weight) idx = self._get_predictions_permutation_indices(indices) - # shape = [BATCH, N_QUERIES] + # shape = (batch_size, num_queries) target_classes_o = torch.cat([target[j] for target, (_, j) in zip(class_labels, indices)]) - # shape = [BATCH, N_QUERIES] + # shape = (batch_size, num_queries) target_classes = torch.full( - (batch_size, num_queries), fill_value=self.num_classes, dtype=torch.int64, device=pred_logits.device + (batch_size, num_queries), fill_value=self.num_labels, dtype=torch.int64, device=pred_logits.device ) target_classes[idx] = target_classes_o - # target_classes is a [BATCH, CLASSES, N_QUERIES], we need to permute pred_logits "b q c -> b c q" - pred_logits_permuted = pred_logits.permute(0, 2, 1) - loss_ce = criterion(pred_logits_permuted, target_classes) + # target_classes is a (batch_size, num_labels, num_queries), we need to permute pred_logits "b q c -> b c q" + pred_logits_transposed = pred_logits.transpose(1, 2) + loss_ce = criterion(pred_logits_transposed, target_classes) losses = {"loss_cross_entropy": loss_ce} return losses def loss_masks( - self, masks_queries_logits: Tensor, mask_labels: Tensor, indices: Tuple[np.array], num_masks: int + self, masks_queries_logits: Tensor, mask_labels: List[Tensor], indices: Tuple[np.array], num_masks: int ) -> Dict[str, Tensor]: """Compute the losses related to the masks using focal and dice loss. @@ -1766,7 +1793,7 @@ def loss_masks( masks_queries_logits (`torch.Tensor`): A tensor of shape `batch_size, num_queries, height, width` mask_labels (`torch.Tensor`): - A tensor of shape `batch_size, num_queries, height, width` + List of mask labels of shape `(labels, height, width)`. indices (`Tuple[np.array])`: The indices computed by the Hungarian matcher. num_masks (`int)`: @@ -1780,10 +1807,12 @@ def loss_masks( """ src_idx = self._get_predictions_permutation_indices(indices) tgt_idx = self._get_targets_permutation_indices(indices) - pred_masks = masks_queries_logits # shape [BATCH, NUM_QUERIES, H, W] - pred_masks = pred_masks[src_idx] # shape [BATCH * NUM_QUERIES, H, W] - target_masks = mask_labels # shape [BATCH, NUM_QUERIES, H, W] - target_masks = target_masks[tgt_idx] # shape [BATCH * NUM_QUERIES, H, W] + # shape (batch_size * num_queries, height, width) + pred_masks = masks_queries_logits[src_idx] + # shape (batch_size, num_queries, height, width) + # pad all and stack the targets to the num_labels dimension + target_masks, _ = self._pad_images_to_max_in_batch(mask_labels) + target_masks = target_masks[tgt_idx] # upsample predictions to the target size, we have to add one dim to use interpolate pred_masks = nn.functional.interpolate( pred_masks[:, None], size=target_masks.shape[-2:], mode="bilinear", align_corners=False @@ -1791,7 +1820,6 @@ def loss_masks( pred_masks = pred_masks[:, 0].flatten(1) target_masks = target_masks.flatten(1) - target_masks = target_masks.view(pred_masks.shape) losses = { "loss_mask": sigmoid_focal_loss(pred_masks, target_masks, num_masks), "loss_dice": dice_loss(pred_masks, target_masks, num_masks), @@ -1810,19 +1838,13 @@ def _get_targets_permutation_indices(self, indices): target_indices = torch.cat([tgt for (_, tgt) in indices]) return batch_indices, target_indices - def get_loss(self, loss, outputs, labels, indices, num_masks): - loss_map = {"labels": self.loss_labels, "masks": self.loss_masks} - if loss not in loss_map: - raise KeyError(f"{loss} not in loss_map") - return loss_map[loss](outputs, labels, indices, num_masks) - def forward( self, - masks_queries_logits: torch.Tensor, - class_queries_logits: torch.Tensor, - mask_labels: torch.Tensor, - class_labels: torch.Tensor, - auxiliary_predictions: Optional[Dict[str, torch.Tensor]] = None, + masks_queries_logits: Tensor, + class_queries_logits: Tensor, + mask_labels: List[Tensor], + class_labels: List[Tensor], + auxiliary_predictions: Optional[Dict[str, Tensor]] = None, ) -> Dict[str, Tensor]: """ This performs the loss computation. @@ -1831,11 +1853,11 @@ def forward( masks_queries_logits (`torch.Tensor`): A tensor of shape `batch_size, num_queries, height, width` class_queries_logits (`torch.Tensor`): - A tensor of shape `batch_size, num_queries, num_classes` + A tensor of shape `batch_size, num_queries, num_labels` mask_labels (`torch.Tensor`): - A tensor of shape `batch_size, num_classes, height, width` - class_labels (`torch.Tensor`): - A tensor of shape `batch_size, num_classes` + List of mask labels of shape `(labels, height, width)`. + class_labels (`List[torch.Tensor]`): + List of class labels of shape `(labels)`. auxiliary_predictions (`Dict[str, torch.Tensor]`, *optional*): if `use_auxiliary_loss` was set to `true` in [`MaskFormerConfig`], then it contains the logits from the inner layers of the Detr's Decoder. @@ -1850,19 +1872,16 @@ def forward( for each auxiliary predictions. """ - # Retrieve the matching between the outputs of the last layer and the labels + # retrieve the matching between the outputs of the last layer and the labels indices = self.matcher(masks_queries_logits, class_queries_logits, mask_labels, class_labels) - - # Compute the average number of target masks accross all nodes, for normalization purposes - num_masks: Number = self.get_num_masks(class_labels, device=class_labels.device) - - # Compute all the requested losses + # compute the average number of target masks for normalization purposes + num_masks: Number = self.get_num_masks(class_labels, device=class_labels[0].device) + # get all the losses losses: Dict[str, Tensor] = { **self.loss_masks(masks_queries_logits, mask_labels, indices, num_masks), **self.loss_labels(class_queries_logits, class_labels, indices), } - - # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. + # in case of auxiliary losses, we repeat this process with the output of each intermediate layer. if auxiliary_predictions is not None: for idx, aux_outputs in enumerate(auxiliary_predictions): masks_queries_logits = aux_outputs["masks_queries_logits"] @@ -1874,8 +1893,10 @@ def forward( return losses def get_num_masks(self, class_labels: torch.Tensor, device: torch.device) -> torch.Tensor: - # Compute the average number of target masks accross all nodes, for normalization purposes - num_masks = class_labels.shape[0] + """ + Computes the average number of target masks accross the batch, for normalization purposes. + """ + num_masks = sum([len(classes) for classes in class_labels]) num_masks_pt = torch.as_tensor([num_masks], dtype=torch.float, device=device) return num_masks_pt @@ -2380,11 +2401,13 @@ def get_loss_dict( loss_dict: Dict[str, Tensor] = self.criterion( masks_queries_logits, class_queries_logits, mask_labels, class_labels, auxiliary_logits ) - # weight each loss by `self.weight_dict[]` - weighted_loss_dict: Dict[str, Tensor] = { - k: v * self.weight_dict[k] for k, v in loss_dict.items() if k in self.weight_dict - } - return weighted_loss_dict + # weight each loss by `self.weight_dict[]` including auxiliary losses + for key, weight in self.weight_dict.items(): + for loss_key, loss in loss_dict.items(): + if key in loss_key: + loss *= weight + + return loss_dict def get_loss(self, loss_dict: Dict[str, Tensor]) -> Tensor: return sum(loss_dict.values()) @@ -2425,8 +2448,8 @@ def get_logits(self, outputs: MaskFormerModelOutput) -> Tuple[Tensor, Tensor, Di def forward( self, pixel_values: Tensor, - mask_labels: Optional[Tensor] = None, - class_labels: Optional[Tensor] = None, + mask_labels: Optional[List[Tensor]] = None, + class_labels: Optional[List[Tensor]] = None, pixel_mask: Optional[Tensor] = None, output_auxiliary_logits: Optional[bool] = None, output_hidden_states: Optional[bool] = None, @@ -2434,10 +2457,11 @@ def forward( return_dict: Optional[bool] = None, ) -> MaskFormerForInstanceSegmentationOutput: r""" - mask_labels (`torch.FloatTensor`, *optional*): - The target mask of shape `(num_classes, height, width)`. - class_labels (`torch.LongTensor`, *optional*): - The target labels of shape `(num_classes)`. + mask_labels (`List[torch.Tensor]`, *optional*): + List of mask labels of shape `(num_labels, height, width)` to be fed to a model + class_labels (`List[torch.LongTensor]`, *optional*): + list of target class labels of shape `(num_labels, height, width)` to be fed to a model. They identify the + labels of `mask_labels`, e.g. the label of `mask_labels[i][j]` if `class_labels[i][j]`. Returns: diff --git a/tests/maskformer/test_feature_extraction_maskformer.py b/tests/maskformer/test_feature_extraction_maskformer.py index c44899ebf502..259954643fc9 100644 --- a/tests/maskformer/test_feature_extraction_maskformer.py +++ b/tests/maskformer/test_feature_extraction_maskformer.py @@ -49,6 +49,9 @@ def __init__( do_normalize=True, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5], + num_labels=10, + reduce_labels=True, + ignore_index=255, ): self.parent = parent self.batch_size = batch_size @@ -68,6 +71,9 @@ def __init__( self.num_classes = 2 self.height = 3 self.width = 4 + self.num_labels = num_labels + self.reduce_labels = reduce_labels + self.ignore_index = ignore_index def prepare_feat_extract_dict(self): return { @@ -78,6 +84,9 @@ def prepare_feat_extract_dict(self): "image_mean": self.image_mean, "image_std": self.image_std, "size_divisibility": self.size_divisibility, + "num_labels": self.num_labels, + "reduce_labels": self.reduce_labels, + "ignore_index": self.ignore_index, } def get_expected_values(self, image_inputs, batched=False): @@ -140,6 +149,8 @@ def test_feat_extract_properties(self): self.assertTrue(hasattr(feature_extractor, "do_resize")) self.assertTrue(hasattr(feature_extractor, "size")) self.assertTrue(hasattr(feature_extractor, "max_size")) + self.assertTrue(hasattr(feature_extractor, "ignore_index")) + self.assertTrue(hasattr(feature_extractor, "num_labels")) def test_batch_feature(self): pass @@ -245,7 +256,9 @@ def test_call_pytorch(self): def test_equivalence_pad_and_create_pixel_mask(self): # Initialize feature_extractors feature_extractor_1 = self.feature_extraction_class(**self.feat_extract_dict) - feature_extractor_2 = self.feature_extraction_class(do_resize=False, do_normalize=False) + feature_extractor_2 = self.feature_extraction_class( + do_resize=False, do_normalize=False, num_labels=self.feature_extract_tester.num_classes + ) # create random PyTorch tensors image_inputs = prepare_image_inputs(self.feature_extract_tester, equal_resolution=False, torchify=True) for image in image_inputs: @@ -262,28 +275,41 @@ def test_equivalence_pad_and_create_pixel_mask(self): torch.allclose(encoded_images_with_method["pixel_mask"], encoded_images["pixel_mask"], atol=1e-4) ) - def comm_get_feature_extractor_inputs(self, with_annotations=False): + def comm_get_feature_extractor_inputs( + self, with_segmentation_maps=False, is_instance_map=False, segmentation_type="np" + ): feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) # prepare image and target - num_classes = 8 batch_size = self.feature_extract_tester.batch_size + num_labels = self.feature_extract_tester.num_labels annotations = None - - if with_annotations: - annotations = [ - { - "masks": np.random.rand(num_classes, 384, 384).astype(np.float32), - "labels": (np.random.rand(num_classes) > 0.5).astype(np.int64), + instance_id_to_semantic_id = None + if with_segmentation_maps: + high = num_labels + if is_instance_map: + high * 2 + labels_expanded = list(range(num_labels)) * 2 + instance_id_to_semantic_id = { + instance_id: label_id for instance_id, label_id in enumerate(labels_expanded) } - for _ in range(batch_size) - ] + annotations = [np.random.randint(0, high, (384, 384)).astype(np.uint8) for _ in range(batch_size)] + if segmentation_type == "pil": + annotations = [Image.fromarray(annotation) for annotation in annotations] image_inputs = prepare_image_inputs(self.feature_extract_tester, equal_resolution=False) - - inputs = feature_extractor(image_inputs, annotations, return_tensors="pt", pad_and_return_pixel_mask=True) + inputs = feature_extractor( + image_inputs, + annotations, + return_tensors="pt", + instance_id_to_semantic_id=instance_id_to_semantic_id, + pad_and_return_pixel_mask=True, + ) return inputs + def test_init_without_params(self): + pass + def test_with_size_divisibility(self): size_divisibilities = [8, 16, 32] weird_input_sizes = [(407, 802), (582, 1094)] @@ -297,27 +323,29 @@ def test_with_size_divisibility(self): self.assertTrue((pixel_values.shape[-1] % size_divisibility) == 0) self.assertTrue((pixel_values.shape[-2] % size_divisibility) == 0) - def test_call_with_numpy_annotations(self): - num_classes = 8 - batch_size = self.feature_extract_tester.batch_size - - inputs = self.comm_get_feature_extractor_inputs(with_annotations=True) + def test_call_with_segmentation_maps(self): + def common(is_instance_map=False, segmentation_type=None): + inputs = self.comm_get_feature_extractor_inputs( + with_segmentation_maps=True, is_instance_map=is_instance_map, segmentation_type=segmentation_type + ) - # check the batch_size - for el in inputs.values(): - self.assertEqual(el.shape[0], batch_size) + mask_labels = inputs["mask_labels"] + class_labels = inputs["class_labels"] + pixel_values = inputs["pixel_values"] - pixel_values = inputs["pixel_values"] - mask_labels = inputs["mask_labels"] - class_labels = inputs["class_labels"] + # check the batch_size + for mask_label, class_label in zip(mask_labels, class_labels): + self.assertEqual(mask_label.shape[0], class_label.shape[0]) + # this ensure padding has happened + self.assertEqual(mask_label.shape[1:], pixel_values.shape[2:]) - self.assertEqual(pixel_values.shape[-2], mask_labels.shape[-2]) - self.assertEqual(pixel_values.shape[-1], mask_labels.shape[-1]) - self.assertEqual(mask_labels.shape[1], class_labels.shape[1]) - self.assertEqual(mask_labels.shape[1], num_classes) + common() + common(is_instance_map=True) + common(is_instance_map=False, segmentation_type="pil") + common(is_instance_map=True, segmentation_type="pil") def test_post_process_segmentation(self): - fature_extractor = self.feature_extraction_class() + fature_extractor = self.feature_extraction_class(num_labels=self.feature_extract_tester.num_classes) outputs = self.feature_extract_tester.get_fake_maskformer_outputs() segmentation = fature_extractor.post_process_segmentation(outputs) @@ -340,7 +368,7 @@ def test_post_process_segmentation(self): ) def test_post_process_semantic_segmentation(self): - fature_extractor = self.feature_extraction_class() + fature_extractor = self.feature_extraction_class(num_labels=self.feature_extract_tester.num_classes) outputs = self.feature_extract_tester.get_fake_maskformer_outputs() segmentation = fature_extractor.post_process_semantic_segmentation(outputs) @@ -361,7 +389,7 @@ def test_post_process_semantic_segmentation(self): self.assertEqual(segmentation.shape, (self.feature_extract_tester.batch_size, *target_size)) def test_post_process_panoptic_segmentation(self): - fature_extractor = self.feature_extraction_class() + fature_extractor = self.feature_extraction_class(num_labels=self.feature_extract_tester.num_classes) outputs = self.feature_extract_tester.get_fake_maskformer_outputs() segmentation = fature_extractor.post_process_panoptic_segmentation(outputs, object_mask_threshold=0) diff --git a/tests/maskformer/test_modeling_maskformer.py b/tests/maskformer/test_modeling_maskformer.py index a27fe41a69cf..50dbecb8de43 100644 --- a/tests/maskformer/test_modeling_maskformer.py +++ b/tests/maskformer/test_modeling_maskformer.py @@ -397,18 +397,19 @@ def test_inference_instance_segmentation_head(self): ).to(torch_device) self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3], expected_slice, atol=TOLERANCE)) - def test_with_annotations_and_loss(self): + def test_with_segmentation_maps_and_loss(self): model = MaskFormerForInstanceSegmentation.from_pretrained(self.model_checkpoints).to(torch_device).eval() feature_extractor = self.default_feature_extractor inputs = feature_extractor( [np.zeros((3, 800, 1333)), np.zeros((3, 800, 1333))], - annotations=[ - {"masks": np.random.rand(10, 384, 384).astype(np.float32), "labels": np.zeros(10).astype(np.int64)}, - {"masks": np.random.rand(10, 384, 384).astype(np.float32), "labels": np.zeros(10).astype(np.int64)}, - ], + segmentation_maps=[np.zeros((384, 384)).astype(np.float32), np.zeros((384, 384)).astype(np.float32)], return_tensors="pt", - ).to(torch_device) + ) + + inputs["pixel_values"] = inputs["pixel_values"].to(torch_device) + inputs["mask_labels"] = [el.to(torch_device) for el in inputs["mask_labels"]] + inputs["class_labels"] = [el.to(torch_device) for el in inputs["class_labels"]] with torch.no_grad(): outputs = model(**inputs) From a73281e3e435937d2550579887c37d0fa4fd534b Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 30 Mar 2022 12:33:16 -0700 Subject: [PATCH 28/54] [examples] max samples can't be bigger than the len of dataset (#16501) * [examples] max samples can't be bigger than then len of dataset * do tf and flax --- .../image-captioning/run_image_captioning_flax.py | 9 ++++++--- examples/flax/language-modeling/run_clm_flax.py | 6 ++++-- examples/flax/question-answering/run_qa.py | 15 ++++++++++----- .../flax/summarization/run_summarization_flax.py | 9 ++++++--- .../pytorch/contrastive-image-text/run_clip.py | 9 ++++++--- examples/pytorch/language-modeling/run_clm.py | 6 ++++-- examples/pytorch/language-modeling/run_mlm.py | 6 ++++-- examples/pytorch/language-modeling/run_plm.py | 6 ++++-- examples/pytorch/multiple-choice/run_swag.py | 6 ++++-- examples/pytorch/question-answering/run_qa.py | 15 ++++++++++----- .../question-answering/run_qa_beam_search.py | 15 ++++++++++----- .../pytorch/question-answering/run_seq2seq_qa.py | 15 ++++++++++----- .../pytorch/summarization/run_summarization.py | 9 ++++++--- examples/pytorch/text-classification/run_glue.py | 9 ++++++--- examples/pytorch/text-classification/run_xnli.py | 9 ++++++--- examples/pytorch/token-classification/run_ner.py | 9 ++++++--- examples/pytorch/translation/run_translation.py | 9 ++++++--- .../jax-projects/model_parallel/run_clm_mp.py | 6 ++++-- .../quantization-qdqbert/run_quant_qa.py | 15 ++++++++++----- .../wav2vec2/run_common_voice.py | 3 ++- examples/tensorflow/language-modeling/run_clm.py | 6 ++++-- examples/tensorflow/language-modeling/run_mlm.py | 6 ++++-- examples/tensorflow/multiple-choice/run_swag.py | 6 ++++-- examples/tensorflow/question-answering/run_qa.py | 15 ++++++++++----- .../tensorflow/summarization/run_summarization.py | 6 ++++-- .../tensorflow/translation/run_translation.py | 6 ++++-- 26 files changed, 154 insertions(+), 77 deletions(-) diff --git a/examples/flax/image-captioning/run_image_captioning_flax.py b/examples/flax/image-captioning/run_image_captioning_flax.py index 4e31da6b9ea9..b4b9afe0d305 100644 --- a/examples/flax/image-captioning/run_image_captioning_flax.py +++ b/examples/flax/image-captioning/run_image_captioning_flax.py @@ -613,7 +613,8 @@ def preprocess_fn(examples, max_target_length, check_image=True): raise ValueError("--do_train requires a train dataset") train_dataset = dataset["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # remove problematic examples # (if feature extraction is performed at the beginning, the filtering is done during preprocessing below # instead here.) @@ -646,7 +647,8 @@ def preprocess_fn(examples, max_target_length, check_image=True): raise ValueError("--do_eval requires a validation dataset") eval_dataset = dataset["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) # remove problematic examples # (if feature extraction is performed at the beginning, the filtering is done during preprocessing below # instead here.) @@ -675,7 +677,8 @@ def preprocess_fn(examples, max_target_length, check_image=True): raise ValueError("--do_predict requires a test dataset") predict_dataset = dataset["test"] if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) # remove problematic examples # (if feature extraction is performed at the beginning, the filtering is done during preprocessing below # instead here.) diff --git a/examples/flax/language-modeling/run_clm_flax.py b/examples/flax/language-modeling/run_clm_flax.py index bbcbc8268bd7..82a9757d5c26 100755 --- a/examples/flax/language-modeling/run_clm_flax.py +++ b/examples/flax/language-modeling/run_clm_flax.py @@ -527,14 +527,16 @@ def group_texts(examples): raise ValueError("--do_train requires a train dataset") train_dataset = lm_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if training_args.do_eval: if "validation" not in tokenized_datasets: raise ValueError("--do_eval requires a validation dataset") eval_dataset = lm_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) # Enable tensorboard only on the master node has_tensorboard = is_tensorboard_available() diff --git a/examples/flax/question-answering/run_qa.py b/examples/flax/question-answering/run_qa.py index 5d21e882e598..a15cca6607cc 100644 --- a/examples/flax/question-answering/run_qa.py +++ b/examples/flax/question-answering/run_qa.py @@ -602,7 +602,8 @@ def prepare_train_features(examples): train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: # We will select sample from whole data if agument is specified - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Create train feature from dataset train_dataset = train_dataset.map( prepare_train_features, @@ -613,7 +614,8 @@ def prepare_train_features(examples): ) if data_args.max_train_samples is not None: # Number of samples might increase during Feature Creation, We select only specified max samples - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) processed_raw_datasets["train"] = train_dataset # Validation preprocessing @@ -669,7 +671,8 @@ def prepare_validation_features(examples): eval_examples = raw_datasets["validation"] if data_args.max_eval_samples is not None: # We will select sample from whole data - eval_examples = eval_examples.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_examples), data_args.max_eval_samples) + eval_examples = eval_examples.select(range(max_eval_samples)) # Validation Feature Creation eval_dataset = eval_examples.map( prepare_validation_features, @@ -680,7 +683,8 @@ def prepare_validation_features(examples): ) if data_args.max_eval_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) processed_raw_datasets["validation"] = eval_dataset if training_args.do_predict: @@ -700,7 +704,8 @@ def prepare_validation_features(examples): ) if data_args.max_predict_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) processed_raw_datasets["test"] = predict_dataset # endregion diff --git a/examples/flax/summarization/run_summarization_flax.py b/examples/flax/summarization/run_summarization_flax.py index cfcf2e63dfce..effe3b58839f 100644 --- a/examples/flax/summarization/run_summarization_flax.py +++ b/examples/flax/summarization/run_summarization_flax.py @@ -547,7 +547,8 @@ def preprocess_function(examples): raise ValueError("--do_train requires a train dataset") train_dataset = dataset["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) train_dataset = train_dataset.map( preprocess_function, batched=True, @@ -563,7 +564,8 @@ def preprocess_function(examples): raise ValueError("--do_eval requires a validation dataset") eval_dataset = dataset["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) eval_dataset = eval_dataset.map( preprocess_function, batched=True, @@ -579,7 +581,8 @@ def preprocess_function(examples): raise ValueError("--do_predict requires a test dataset") predict_dataset = dataset["test"] if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) predict_dataset = predict_dataset.map( preprocess_function, batched=True, diff --git a/examples/pytorch/contrastive-image-text/run_clip.py b/examples/pytorch/contrastive-image-text/run_clip.py index 8a2ad8c55066..79fd123064a1 100644 --- a/examples/pytorch/contrastive-image-text/run_clip.py +++ b/examples/pytorch/contrastive-image-text/run_clip.py @@ -404,7 +404,8 @@ def filter_corrupt_images(examples): raise ValueError("--do_train requires a train dataset") train_dataset = dataset["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) train_dataset = train_dataset.filter( filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers @@ -426,7 +427,8 @@ def filter_corrupt_images(examples): raise ValueError("--do_eval requires a train validation") eval_dataset = dataset["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) eval_dataset = eval_dataset.filter( filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers @@ -448,7 +450,8 @@ def filter_corrupt_images(examples): raise ValueError("--do_predict requires a test dataset") test_dataset = dataset["test"] if data_args.max_eval_samples is not None: - test_dataset = test_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(test_dataset), data_args.max_eval_samples) + test_dataset = test_dataset.select(range(max_eval_samples)) test_dataset = test_dataset.filter( filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers diff --git a/examples/pytorch/language-modeling/run_clm.py b/examples/pytorch/language-modeling/run_clm.py index 5534e6901fb6..a1cdcf9ee4a9 100755 --- a/examples/pytorch/language-modeling/run_clm.py +++ b/examples/pytorch/language-modeling/run_clm.py @@ -445,14 +445,16 @@ def group_texts(examples): raise ValueError("--do_train requires a train dataset") train_dataset = lm_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if training_args.do_eval: if "validation" not in tokenized_datasets: raise ValueError("--do_eval requires a validation dataset") eval_dataset = lm_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) def preprocess_logits_for_metrics(logits, labels): if isinstance(logits, tuple): diff --git a/examples/pytorch/language-modeling/run_mlm.py b/examples/pytorch/language-modeling/run_mlm.py index 7ceae8b17a8c..6ea3c2c934d3 100755 --- a/examples/pytorch/language-modeling/run_mlm.py +++ b/examples/pytorch/language-modeling/run_mlm.py @@ -468,14 +468,16 @@ def group_texts(examples): raise ValueError("--do_train requires a train dataset") train_dataset = tokenized_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if training_args.do_eval: if "validation" not in tokenized_datasets: raise ValueError("--do_eval requires a validation dataset") eval_dataset = tokenized_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) def preprocess_logits_for_metrics(logits, labels): if isinstance(logits, tuple): diff --git a/examples/pytorch/language-modeling/run_plm.py b/examples/pytorch/language-modeling/run_plm.py index 1d7c42ba9c06..d1c09896d8e7 100755 --- a/examples/pytorch/language-modeling/run_plm.py +++ b/examples/pytorch/language-modeling/run_plm.py @@ -438,14 +438,16 @@ def group_texts(examples): raise ValueError("--do_train requires a train dataset") train_dataset = tokenized_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if training_args.do_eval: if "validation" not in tokenized_datasets: raise ValueError("--do_eval requires a validation dataset") eval_dataset = tokenized_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) # Data collator data_collator = DataCollatorForPermutationLanguageModeling( diff --git a/examples/pytorch/multiple-choice/run_swag.py b/examples/pytorch/multiple-choice/run_swag.py index eb9f52f4d54a..01c9e8bcf7d2 100755 --- a/examples/pytorch/multiple-choice/run_swag.py +++ b/examples/pytorch/multiple-choice/run_swag.py @@ -352,7 +352,8 @@ def preprocess_function(examples): raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( preprocess_function, @@ -366,7 +367,8 @@ def preprocess_function(examples): raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( preprocess_function, diff --git a/examples/pytorch/question-answering/run_qa.py b/examples/pytorch/question-answering/run_qa.py index 788cecae683f..67aaf1d84ff0 100755 --- a/examples/pytorch/question-answering/run_qa.py +++ b/examples/pytorch/question-answering/run_qa.py @@ -421,7 +421,8 @@ def prepare_train_features(examples): train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: # We will select sample from whole data if argument is specified - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Create train feature from dataset with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( @@ -434,7 +435,8 @@ def prepare_train_features(examples): ) if data_args.max_train_samples is not None: # Number of samples might increase during Feature Creation, We select only specified max samples - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Validation preprocessing def prepare_validation_features(examples): @@ -489,7 +491,8 @@ def prepare_validation_features(examples): eval_examples = raw_datasets["validation"] if data_args.max_eval_samples is not None: # We will select sample from whole data - eval_examples = eval_examples.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_examples), data_args.max_eval_samples) + eval_examples = eval_examples.select(range(max_eval_samples)) # Validation Feature Creation with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_examples.map( @@ -502,7 +505,8 @@ def prepare_validation_features(examples): ) if data_args.max_eval_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) if training_args.do_predict: if "test" not in raw_datasets: @@ -523,7 +527,8 @@ def prepare_validation_features(examples): ) if data_args.max_predict_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) # Data collator # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data diff --git a/examples/pytorch/question-answering/run_qa_beam_search.py b/examples/pytorch/question-answering/run_qa_beam_search.py index 13582c86d1b2..4c79be08b91b 100755 --- a/examples/pytorch/question-answering/run_qa_beam_search.py +++ b/examples/pytorch/question-answering/run_qa_beam_search.py @@ -432,7 +432,8 @@ def prepare_train_features(examples): train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: # Select samples from Dataset, This will help to decrease processing time - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Create Training Features with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( @@ -445,7 +446,8 @@ def prepare_train_features(examples): ) if data_args.max_train_samples is not None: # Select samples from dataset again since Feature Creation might increase number of features - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Validation preprocessing def prepare_validation_features(examples): @@ -519,7 +521,8 @@ def prepare_validation_features(examples): eval_examples = raw_datasets["validation"] if data_args.max_eval_samples is not None: # Selecting Eval Samples from Dataset - eval_examples = eval_examples.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_examples), data_args.max_eval_samples) + eval_examples = eval_examples.select(range(max_eval_samples)) # Create Features from Eval Dataset with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_examples.map( @@ -532,7 +535,8 @@ def prepare_validation_features(examples): ) if data_args.max_eval_samples is not None: # Selecting Samples from Dataset again since Feature Creation might increase samples size - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) if training_args.do_predict: if "test" not in raw_datasets: @@ -553,7 +557,8 @@ def prepare_validation_features(examples): ) if data_args.max_predict_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) # Data collator # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data diff --git a/examples/pytorch/question-answering/run_seq2seq_qa.py b/examples/pytorch/question-answering/run_seq2seq_qa.py index 071902a2ab5c..b4434102599c 100644 --- a/examples/pytorch/question-answering/run_seq2seq_qa.py +++ b/examples/pytorch/question-answering/run_seq2seq_qa.py @@ -489,7 +489,8 @@ def preprocess_validation_function(examples): train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: # We will select sample from whole data if agument is specified - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Create train feature from dataset with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( @@ -502,7 +503,8 @@ def preprocess_validation_function(examples): ) if data_args.max_train_samples is not None: # Number of samples might increase during Feature Creation, We select only specified max samples - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if training_args.do_eval: if "validation" not in raw_datasets: @@ -510,7 +512,8 @@ def preprocess_validation_function(examples): eval_examples = raw_datasets["validation"] if data_args.max_eval_samples is not None: # We will select sample from whole data - eval_examples = eval_examples.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_examples), data_args.max_eval_samples) + eval_examples = eval_examples.select(range(max_eval_samples)) # Validation Feature Creation with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_examples.map( @@ -523,7 +526,8 @@ def preprocess_validation_function(examples): ) if data_args.max_eval_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) if training_args.do_predict: if "test" not in raw_datasets: @@ -544,7 +548,8 @@ def preprocess_validation_function(examples): ) if data_args.max_predict_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) # Data collator label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id diff --git a/examples/pytorch/summarization/run_summarization.py b/examples/pytorch/summarization/run_summarization.py index 8d8e355e4f45..66aeb981bdf4 100755 --- a/examples/pytorch/summarization/run_summarization.py +++ b/examples/pytorch/summarization/run_summarization.py @@ -504,7 +504,8 @@ def preprocess_function(examples): raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( preprocess_function, @@ -521,7 +522,8 @@ def preprocess_function(examples): raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( preprocess_function, @@ -538,7 +540,8 @@ def preprocess_function(examples): raise ValueError("--do_predict requires a test dataset") predict_dataset = raw_datasets["test"] if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) with training_args.main_process_first(desc="prediction dataset map pre-processing"): predict_dataset = predict_dataset.map( preprocess_function, diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py index 5f7ba4ca48c3..88be878faea2 100755 --- a/examples/pytorch/text-classification/run_glue.py +++ b/examples/pytorch/text-classification/run_glue.py @@ -415,21 +415,24 @@ def preprocess_function(examples): raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if training_args.do_eval: if "validation" not in raw_datasets and "validation_matched" not in raw_datasets: raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation_matched" if data_args.task_name == "mnli" else "validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) if training_args.do_predict or data_args.task_name is not None or data_args.test_file is not None: if "test" not in raw_datasets and "test_matched" not in raw_datasets: raise ValueError("--do_predict requires a test dataset") predict_dataset = raw_datasets["test_matched" if data_args.task_name == "mnli" else "test"] if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) # Log a few random samples from the training set: if training_args.do_train: diff --git a/examples/pytorch/text-classification/run_xnli.py b/examples/pytorch/text-classification/run_xnli.py index 866d69687175..f54b1ec2aa60 100755 --- a/examples/pytorch/text-classification/run_xnli.py +++ b/examples/pytorch/text-classification/run_xnli.py @@ -279,7 +279,8 @@ def preprocess_function(examples): if training_args.do_train: if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( preprocess_function, @@ -293,7 +294,8 @@ def preprocess_function(examples): if training_args.do_eval: if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( preprocess_function, @@ -304,7 +306,8 @@ def preprocess_function(examples): if training_args.do_predict: if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) with training_args.main_process_first(desc="prediction dataset map pre-processing"): predict_dataset = predict_dataset.map( preprocess_function, diff --git a/examples/pytorch/token-classification/run_ner.py b/examples/pytorch/token-classification/run_ner.py index ca08efceb5c7..9ff64b37978c 100755 --- a/examples/pytorch/token-classification/run_ner.py +++ b/examples/pytorch/token-classification/run_ner.py @@ -431,7 +431,8 @@ def tokenize_and_align_labels(examples): raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( tokenize_and_align_labels, @@ -446,7 +447,8 @@ def tokenize_and_align_labels(examples): raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( tokenize_and_align_labels, @@ -461,7 +463,8 @@ def tokenize_and_align_labels(examples): raise ValueError("--do_predict requires a test dataset") predict_dataset = raw_datasets["test"] if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) with training_args.main_process_first(desc="prediction dataset map pre-processing"): predict_dataset = predict_dataset.map( tokenize_and_align_labels, diff --git a/examples/pytorch/translation/run_translation.py b/examples/pytorch/translation/run_translation.py index ef10ff1890b9..b458a3f0cd65 100755 --- a/examples/pytorch/translation/run_translation.py +++ b/examples/pytorch/translation/run_translation.py @@ -433,7 +433,8 @@ def preprocess_function(examples): raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( preprocess_function, @@ -450,7 +451,8 @@ def preprocess_function(examples): raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( preprocess_function, @@ -467,7 +469,8 @@ def preprocess_function(examples): raise ValueError("--do_predict requires a test dataset") predict_dataset = raw_datasets["test"] if data_args.max_predict_samples is not None: - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) with training_args.main_process_first(desc="prediction dataset map pre-processing"): predict_dataset = predict_dataset.map( preprocess_function, diff --git a/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py b/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py index c56f10478f56..3371dc3bd4df 100644 --- a/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py +++ b/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py @@ -398,14 +398,16 @@ def group_texts(examples): raise ValueError("--do_train requires a train dataset") train_dataset = lm_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if training_args.do_eval: if "validation" not in tokenized_datasets: raise ValueError("--do_eval requires a validation dataset") eval_dataset = lm_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) # Enable tensorboard only on the master node has_tensorboard = is_tensorboard_available() diff --git a/examples/research_projects/quantization-qdqbert/run_quant_qa.py b/examples/research_projects/quantization-qdqbert/run_quant_qa.py index 01791681eff9..36bfb45c8ffc 100755 --- a/examples/research_projects/quantization-qdqbert/run_quant_qa.py +++ b/examples/research_projects/quantization-qdqbert/run_quant_qa.py @@ -434,7 +434,8 @@ def prepare_train_features(examples): train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: # We will select sample from whole data if agument is specified - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Create train feature from dataset with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( @@ -447,7 +448,8 @@ def prepare_train_features(examples): ) if data_args.max_train_samples is not None: # Number of samples might increase during Feature Creation, We select only specified max samples - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Validation preprocessing def prepare_validation_features(examples): @@ -497,7 +499,8 @@ def prepare_validation_features(examples): eval_examples = raw_datasets["validation"] if data_args.max_eval_samples is not None: # We will select sample from whole data - eval_examples = eval_examples.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_examples), data_args.max_eval_samples) + eval_examples = eval_examples.select(range(max_eval_samples)) # Validation Feature Creation with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_examples.map( @@ -510,7 +513,8 @@ def prepare_validation_features(examples): ) if data_args.max_eval_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) if training_args.do_predict: if "test" not in raw_datasets: @@ -531,7 +535,8 @@ def prepare_validation_features(examples): ) if data_args.max_predict_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) # Data collator # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data diff --git a/examples/research_projects/wav2vec2/run_common_voice.py b/examples/research_projects/wav2vec2/run_common_voice.py index edae86641e0b..5825c1feb10b 100644 --- a/examples/research_projects/wav2vec2/run_common_voice.py +++ b/examples/research_projects/wav2vec2/run_common_voice.py @@ -375,7 +375,8 @@ def extract_all_chars(batch): ) if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if data_args.max_val_samples is not None: eval_dataset = eval_dataset.select(range(data_args.max_val_samples)) diff --git a/examples/tensorflow/language-modeling/run_clm.py b/examples/tensorflow/language-modeling/run_clm.py index 0aad9949542f..4cbc00b3cdc9 100755 --- a/examples/tensorflow/language-modeling/run_clm.py +++ b/examples/tensorflow/language-modeling/run_clm.py @@ -415,9 +415,11 @@ def group_texts(examples): train_dataset = train_dataset.select(train_indices) if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) # Log a few random samples from the training set: for index in random.sample(range(len(train_dataset)), 3): diff --git a/examples/tensorflow/language-modeling/run_mlm.py b/examples/tensorflow/language-modeling/run_mlm.py index b2e6a487e93e..44c5d230318b 100755 --- a/examples/tensorflow/language-modeling/run_mlm.py +++ b/examples/tensorflow/language-modeling/run_mlm.py @@ -456,9 +456,11 @@ def group_texts(examples): train_dataset = train_dataset.select(train_indices) if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) # Log a few random samples from the training set: for index in random.sample(range(len(train_dataset)), 3): diff --git a/examples/tensorflow/multiple-choice/run_swag.py b/examples/tensorflow/multiple-choice/run_swag.py index 0c1c62de26fa..e14815cf81f3 100644 --- a/examples/tensorflow/multiple-choice/run_swag.py +++ b/examples/tensorflow/multiple-choice/run_swag.py @@ -369,7 +369,8 @@ def preprocess_function(examples): train_dataset = raw_datasets["train"] non_label_columns = [feature for feature in train_dataset.features if feature not in ("label", "labels")] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( preprocess_function, @@ -385,7 +386,8 @@ def preprocess_function(examples): if not training_args.do_train: non_label_columns = [feature for feature in eval_dataset.features if feature not in ("label", "labels")] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( preprocess_function, diff --git a/examples/tensorflow/question-answering/run_qa.py b/examples/tensorflow/question-answering/run_qa.py index 39437da46788..50e8c7f50d96 100755 --- a/examples/tensorflow/question-answering/run_qa.py +++ b/examples/tensorflow/question-answering/run_qa.py @@ -438,7 +438,8 @@ def prepare_train_features(examples): train_dataset = datasets["train"] if data_args.max_train_samples is not None: # We will select sample from whole data if agument is specified - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) # Create train feature from dataset train_dataset = train_dataset.map( prepare_train_features, @@ -449,7 +450,8 @@ def prepare_train_features(examples): ) if data_args.max_train_samples is not None: # Number of samples might increase during Feature Creation, We select only specified max samples - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) processed_datasets["train"] = train_dataset # Validation preprocessing @@ -505,7 +507,8 @@ def prepare_validation_features(examples): eval_examples = datasets["validation"] if data_args.max_eval_samples is not None: # We will select sample from whole data - eval_examples = eval_examples.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_examples), data_args.max_eval_samples) + eval_examples = eval_examples.select(range(max_eval_samples)) # Validation Feature Creation eval_dataset = eval_examples.map( prepare_validation_features, @@ -516,7 +519,8 @@ def prepare_validation_features(examples): ) if data_args.max_eval_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) processed_datasets["validation"] = eval_dataset if training_args.do_predict: @@ -536,7 +540,8 @@ def prepare_validation_features(examples): ) if data_args.max_predict_samples is not None: # During Feature creation dataset samples might increase, we will select required samples again - predict_dataset = predict_dataset.select(range(data_args.max_predict_samples)) + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) processed_datasets["test"] = predict_dataset # endregion diff --git a/examples/tensorflow/summarization/run_summarization.py b/examples/tensorflow/summarization/run_summarization.py index 004f25157f06..e40c763530c0 100644 --- a/examples/tensorflow/summarization/run_summarization.py +++ b/examples/tensorflow/summarization/run_summarization.py @@ -490,7 +490,8 @@ def preprocess_function(examples): raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( preprocess_function, @@ -509,7 +510,8 @@ def preprocess_function(examples): raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( preprocess_function, diff --git a/examples/tensorflow/translation/run_translation.py b/examples/tensorflow/translation/run_translation.py index ef997a6ed25f..fce150b712ad 100644 --- a/examples/tensorflow/translation/run_translation.py +++ b/examples/tensorflow/translation/run_translation.py @@ -445,7 +445,8 @@ def preprocess_function(examples): raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: - train_dataset = train_dataset.select(range(data_args.max_train_samples)) + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) with training_args.main_process_first(desc="train dataset map pre-processing"): train_dataset = train_dataset.map( preprocess_function, @@ -464,7 +465,8 @@ def preprocess_function(examples): raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation"] if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) with training_args.main_process_first(desc="validation dataset map pre-processing"): eval_dataset = eval_dataset.map( preprocess_function, From 81ac45f85c35244831f11f73c09ea10eee4f953a Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Wed, 30 Mar 2022 13:28:35 -0700 Subject: [PATCH 29/54] update smddp api to v1.4.0 (#16371) * update smddp api to v1.4.0 * Update src/transformers/trainer.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update src/transformers/trainer.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * address comments * fix style * remove unused import * fix indent * disable style check for import * fix space Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/sagemaker/training_args_sm.py | 6 +++--- src/transformers/trainer.py | 10 ++++------ src/transformers/trainer_pt_utils.py | 16 +++------------- src/transformers/trainer_utils.py | 5 ----- src/transformers/training_args.py | 17 ++++++++--------- 5 files changed, 18 insertions(+), 36 deletions(-) diff --git a/src/transformers/sagemaker/training_args_sm.py b/src/transformers/sagemaker/training_args_sm.py index f6c57d8f8577..7328a831b979 100644 --- a/src/transformers/sagemaker/training_args_sm.py +++ b/src/transformers/sagemaker/training_args_sm.py @@ -90,10 +90,10 @@ def _setup_devices(self) -> "torch.device": device = torch.device("cuda", local_rank) self._n_gpu = 1 elif is_sagemaker_dp_enabled(): - import smdistributed.dataparallel.torch.distributed as dist + import smdistributed.dataparallel.torch.torch_smddp # noqa: F401 - dist.init_process_group() - self.local_rank = dist.get_local_rank() + torch.distributed.init_process_group(backend="smddp") + self.local_rank = int(os.getenv("SMDATAPARALLEL_LOCAL_RANK")) device = torch.device("cuda", self.local_rank) self._n_gpu = 1 elif self.local_rank == -1: diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 993e5248635e..157e65d18352 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -51,6 +51,7 @@ import numpy as np import torch +import torch.distributed as dist from packaging import version from torch import nn from torch.utils.data import DataLoader, Dataset, RandomSampler, SequentialSampler @@ -170,11 +171,6 @@ from fairscale.optim import OSS from fairscale.optim.grad_scaler import ShardedGradScaler -if is_sagemaker_dp_enabled(): - import smdistributed.dataparallel.torch.distributed as dist - from smdistributed.dataparallel.torch.parallel.distributed import DistributedDataParallel as DDP -else: - import torch.distributed as dist if is_sagemaker_mp_enabled(): import smdistributed.modelparallel.torch as smp @@ -1078,7 +1074,9 @@ def _wrap_model(self, model, training=True): ).to(self.args.device) elif is_sagemaker_dp_enabled(): - model = DDP(model, device_ids=[dist.get_local_rank()], broadcast_buffers=False) + model = nn.parallel.DistributedDataParallel( + model, device_ids=[int(os.getenv("SMDATAPARALLEL_LOCAL_RANK"))] + ) elif self.args.local_rank != -1: kwargs = {} if self.args.ddp_find_unused_parameters is not None: diff --git a/src/transformers/trainer_pt_utils.py b/src/transformers/trainer_pt_utils.py index 63842d484932..da7e9accaf78 100644 --- a/src/transformers/trainer_pt_utils.py +++ b/src/transformers/trainer_pt_utils.py @@ -29,25 +29,15 @@ import numpy as np import torch +import torch.distributed as dist from packaging import version from torch import nn from torch.utils.data import Dataset, IterableDataset, RandomSampler, Sampler from torch.utils.data.distributed import DistributedSampler from .tokenization_utils_base import BatchEncoding -from .utils import ( - is_sagemaker_dp_enabled, - is_sagemaker_mp_enabled, - is_torch_tpu_available, - is_training_run_on_sagemaker, - logging, -) - - -if is_sagemaker_dp_enabled(): - import smdistributed.dataparallel.torch.distributed as dist -else: - import torch.distributed as dist +from .utils import is_sagemaker_mp_enabled, is_torch_tpu_available, is_training_run_on_sagemaker, logging + if is_training_run_on_sagemaker(): logging.add_handler(StreamHandler(sys.stdout)) diff --git a/src/transformers/trainer_utils.py b/src/transformers/trainer_utils.py index 04f5dc1058ed..d8d15affb9ed 100644 --- a/src/transformers/trainer_utils.py +++ b/src/transformers/trainer_utils.py @@ -32,7 +32,6 @@ from .utils import ( ExplicitEnum, is_psutil_available, - is_sagemaker_dp_enabled, is_tf_available, is_torch_available, is_torch_cuda_available, @@ -263,10 +262,6 @@ def total_processes_number(local_rank): import torch_xla.core.xla_model as xm return xm.xrt_world_size() - elif is_sagemaker_dp_enabled(): - import smdistributed.dataparallel.torch.distributed as dist - - return dist.get_world_size() elif local_rank != -1 and is_torch_available(): import torch diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index b0e6fbc6e85e..6176794ab032 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -41,12 +41,11 @@ if is_torch_available(): import torch + import torch.distributed as dist if is_torch_tpu_available(): import torch_xla.core.xla_model as xm -if is_sagemaker_dp_enabled(): - import smdistributed.dataparallel.torch.distributed as sm_dist if is_sagemaker_mp_enabled(): import smdistributed.modelparallel.torch as smp @@ -1046,8 +1045,8 @@ def _setup_devices(self) -> "torch.device": device = torch.device("cuda", local_rank) self._n_gpu = 1 elif is_sagemaker_dp_enabled(): - sm_dist.init_process_group() - self.local_rank = sm_dist.get_local_rank() + dist.init_process_group(backend="smddp") + self.local_rank = int(os.getenv("SMDATAPARALLEL_LOCAL_RANK")) device = torch.device("cuda", self.local_rank) self._n_gpu = 1 elif self.deepspeed: @@ -1149,7 +1148,7 @@ def world_size(self): elif is_sagemaker_mp_enabled(): return smp.dp_size() if not smp.state.cfg.prescaled_batch else smp.rdp_size() elif is_sagemaker_dp_enabled(): - return sm_dist.get_world_size() + return dist.get_world_size() elif self.local_rank != -1: return torch.distributed.get_world_size() return 1 @@ -1165,7 +1164,7 @@ def process_index(self): elif is_sagemaker_mp_enabled(): return smp.dp_rank() if not smp.state.cfg.prescaled_batch else smp.rdp_rank() elif is_sagemaker_dp_enabled(): - return sm_dist.get_rank() + return dist.get_rank() elif self.local_rank != -1: return torch.distributed.get_rank() return 0 @@ -1181,7 +1180,7 @@ def local_process_index(self): elif is_sagemaker_mp_enabled(): return smp.local_rank() elif is_sagemaker_dp_enabled(): - return sm_dist.get_rank() + return dist.get_rank() elif self.local_rank != -1: return self.local_rank return 0 @@ -1281,7 +1280,7 @@ def main_process_first(self, local=True, desc="work"): if is_torch_tpu_available(): xm.rendezvous(desc) elif is_sagemaker_dp_enabled(): - sm_dist.barrier() + dist.barrier() else: torch.distributed.barrier() yield @@ -1292,7 +1291,7 @@ def main_process_first(self, local=True, desc="work"): if is_torch_tpu_available(): xm.rendezvous(desc) elif is_sagemaker_dp_enabled(): - sm_dist.barrier() + dist.barrier() else: torch.distributed.barrier() else: From 857eb87cc42276edb537296b85021a8f60a0b3be Mon Sep 17 00:00:00 2001 From: "Manuel R. Ciosici" Date: Wed, 30 Mar 2022 14:12:29 -0700 Subject: [PATCH 30/54] Support reduce_bucket_size=auto for deepspeed stages <3 (#16496) --- src/transformers/deepspeed.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/deepspeed.py b/src/transformers/deepspeed.py index 8fc0d6a0b6a0..46cfb9730760 100644 --- a/src/transformers/deepspeed.py +++ b/src/transformers/deepspeed.py @@ -261,13 +261,13 @@ def trainer_config_finalize(self, args, model, num_training_steps): """ This stage is run after we have the model and know num_training_steps. - Now we we can complete the configuration process. + Now we can complete the configuration process. """ # zero + hidden_size = model.config.hidden_size + self.fill_only("zero_optimization.reduce_bucket_size", hidden_size * hidden_size) if self.is_zero3(): # automatically assign the optimal config values based on model config - hidden_size = model.config.hidden_size - self.fill_only("zero_optimization.reduce_bucket_size", hidden_size * hidden_size) self.fill_only("zero_optimization.stage3_prefetch_bucket_size", 0.9 * hidden_size * hidden_size) self.fill_only("zero_optimization.stage3_param_persistence_threshold", 10 * hidden_size) From a8b6443e065a14af755f8f0ad1d3b5d70cc5f3ed Mon Sep 17 00:00:00 2001 From: Francesco Saverio Zuppichini Date: Thu, 31 Mar 2022 09:32:33 +0200 Subject: [PATCH 31/54] Refactor Modeling Outputs (#16341) * first proposal * replace model outputs in various models * conflicts * docstring * update poolformer * minor change in docstring * CI * removed poolformer specific outputs from doc * removed convnext specific outputs from doc * CI * weird char in segformer * conversations * reverted docstring for BaseModelOutputWithPooling * update outputs * changed docstring in BaseModelOutput * updated docstring in modeling outputs * typos :) * fixed typo after copy & paste it all around * CI * Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> * segformer Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- docs/source/model_doc/convnext.mdx | 5 - docs/source/model_doc/poolformer.mdx | 6 - src/transformers/modeling_outputs.py | 212 +++++++++++------- src/transformers/models/beit/modeling_beit.py | 8 +- .../models/convnext/modeling_convnext.py | 85 +------ src/transformers/models/deit/modeling_deit.py | 8 +- .../models/poolformer/modeling_poolformer.py | 67 +----- .../models/resnet/modeling_resnet.py | 66 ++---- .../models/segformer/modeling_segformer.py | 35 ++- src/transformers/models/van/modeling_van.py | 82 +------ src/transformers/models/vit/modeling_vit.py | 8 +- 11 files changed, 226 insertions(+), 356 deletions(-) diff --git a/docs/source/model_doc/convnext.mdx b/docs/source/model_doc/convnext.mdx index 4d46248565f9..732e0eb7a59f 100644 --- a/docs/source/model_doc/convnext.mdx +++ b/docs/source/model_doc/convnext.mdx @@ -40,11 +40,6 @@ alt="drawing" width="600"/> This model was contributed by [nielsr](https://huggingface.co/nielsr). TensorFlow version of the model was contributed by [ariG23498](https://github.com/ariG23498), [gante](https://github.com/gante), and [sayakpaul](https://github.com/sayakpaul) (equal contribution). The original code can be found [here](https://github.com/facebookresearch/ConvNeXt). -## ConvNeXT specific outputs - -[[autodoc]] models.convnext.modeling_convnext.ConvNextModelOutput - - ## ConvNextConfig [[autodoc]] ConvNextConfig diff --git a/docs/source/model_doc/poolformer.mdx b/docs/source/model_doc/poolformer.mdx index 78101005edc8..ac06bb63dbce 100644 --- a/docs/source/model_doc/poolformer.mdx +++ b/docs/source/model_doc/poolformer.mdx @@ -41,12 +41,6 @@ Tips: This model was contributed by [heytanay](https://huggingface.co/heytanay). The original code can be found [here](https://github.com/sail-sg/poolformer). -## PoolFormer specific outputs - -[[autodoc]] models.poolformer.modeling_poolformer.PoolFormerModelOutput - -[[autodoc]] models.poolformer.modeling_poolformer.PoolFormerClassifierOutput - ## PoolFormerConfig [[autodoc]] PoolFormerConfig diff --git a/src/transformers/modeling_outputs.py b/src/transformers/modeling_outputs.py index 1360e2870fad..8a08c0b3986e 100644 --- a/src/transformers/modeling_outputs.py +++ b/src/transformers/modeling_outputs.py @@ -29,10 +29,10 @@ class BaseModelOutput(ModelOutput): last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -46,6 +46,25 @@ class BaseModelOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None +@dataclass +class BaseModelOutputWithNoAttention(ModelOutput): + """ + Base class for model's outputs, with potential hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, num_channels, height, width)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + """ + + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + + @dataclass class BaseModelOutputWithPooling(ModelOutput): """ @@ -60,10 +79,10 @@ class BaseModelOutputWithPooling(ModelOutput): the classification token after processing through a linear layer and a tanh activation function. The linear layer weights are trained from the next sentence prediction (classification) objective during pretraining. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -78,6 +97,28 @@ class BaseModelOutputWithPooling(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None +@dataclass +class BaseModelOutputWithPoolingAndNoAttention(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): + Last layer hidden-state after a pooling operation on the spatial dimensions. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, num_channels, height, width)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + """ + + last_hidden_state: torch.FloatTensor = None + pooler_output: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + + @dataclass class BaseModelOutputWithPast(ModelOutput): """ @@ -99,10 +140,10 @@ class BaseModelOutputWithPast(ModelOutput): `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -126,10 +167,10 @@ class BaseModelOutputWithCrossAttentions(ModelOutput): last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -164,10 +205,10 @@ class BaseModelOutputWithPoolingAndCrossAttentions(ModelOutput): the classification token after processing through a linear layer and a tanh activation function. The linear layer weights are trained from the next sentence prediction (classification) objective during pretraining. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -220,10 +261,10 @@ class BaseModelOutputWithPastAndCrossAttentions(ModelOutput): `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -265,10 +306,10 @@ class Seq2SeqModelOutput(ModelOutput): Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the decoder at the output of each layer plus the initial embedding outputs. + Hidden-states of the decoder at the output of each layer plus the optional initial embedding outputs. decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -284,10 +325,10 @@ class Seq2SeqModelOutput(ModelOutput): encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder of the model. encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the encoder at the output of each layer plus the initial embedding outputs. + Hidden-states of the encoder at the output of each layer plus the optional initial embedding outputs. encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -317,10 +358,10 @@ class CausalLMOutput(ModelOutput): logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -352,10 +393,10 @@ class CausalLMOutputWithPast(ModelOutput): Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -382,10 +423,10 @@ class CausalLMOutputWithCrossAttentions(ModelOutput): logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -432,10 +473,10 @@ class SequenceClassifierOutputWithPast(ModelOutput): Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -462,10 +503,10 @@ class MaskedLMOutput(ModelOutput): logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -498,8 +539,8 @@ class Seq2SeqLMOutput(ModelOutput): Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of each layer plus the initial embedding outputs. decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): @@ -517,8 +558,8 @@ class Seq2SeqLMOutput(ModelOutput): encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder of the model. encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of each layer plus the initial embedding outputs. encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): @@ -552,10 +593,10 @@ class NextSentencePredictorOutput(ModelOutput): Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -581,10 +622,10 @@ class SequenceClassifierOutput(ModelOutput): logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`): Classification (or regression if config.num_labels==1) scores (before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -617,8 +658,8 @@ class Seq2SeqSequenceClassifierOutput(ModelOutput): Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of each layer plus the initial embedding outputs. decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): @@ -636,8 +677,8 @@ class Seq2SeqSequenceClassifierOutput(ModelOutput): encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder of the model. encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of each layer plus the initial embedding outputs. encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): @@ -672,10 +713,10 @@ class MultipleChoiceModelOutput(ModelOutput): Classification scores (before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -701,10 +742,10 @@ class TokenClassifierOutput(ModelOutput): logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`): Classification scores (before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -732,10 +773,10 @@ class QuestionAnsweringModelOutput(ModelOutput): end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Span-end scores (before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -771,8 +812,8 @@ class Seq2SeqQuestionAnsweringModelOutput(ModelOutput): Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of each layer plus the initial embedding outputs. decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): @@ -790,8 +831,8 @@ class Seq2SeqQuestionAnsweringModelOutput(ModelOutput): encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder of the model. encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of each layer plus the initial embedding outputs. encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): @@ -834,10 +875,10 @@ class SemanticSegmenterOutput(ModelOutput): hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, patch_size, hidden_size)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, patch_size, hidden_size)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, sequence_length)`. @@ -863,9 +904,9 @@ class ImageClassifierOutput(ModelOutput): logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`): Classification (or regression if config.num_labels==1) scores (before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of - shape `(batch_size, num_channels, height, width)`. Hidden-states (also called feature maps) of the model at - the output of each stage. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each stage) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states + (also called feature maps) of the model at the output of each stage. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, sequence_length)`. @@ -880,6 +921,27 @@ class ImageClassifierOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None +@dataclass +class ImageClassifierOutputWithNoAttention(ModelOutput): + """ + Base class for outputs of image classification models. + + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Classification (or regression if config.num_labels==1) loss. + logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`): + Classification (or regression if config.num_labels==1) scores (before SoftMax). + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each stage) of shape `(batch_size, num_channels, height, width)`. Hidden-states (also + called feature maps) of the model at the output of each stage. + """ + + loss: Optional[torch.FloatTensor] = None + logits: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + + @dataclass class DepthEstimatorOutput(ModelOutput): """ @@ -892,10 +954,10 @@ class DepthEstimatorOutput(ModelOutput): Predicted depth for each pixel. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, num_channels, height, width)`. + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, num_channels, height, width)`. - Hidden-states of the model at the output of each layer plus the initial embedding outputs. + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, sequence_length)`. diff --git a/src/transformers/models/beit/modeling_beit.py b/src/transformers/models/beit/modeling_beit.py index 6822f2e0ba2d..3422afcce5d3 100755 --- a/src/transformers/models/beit/modeling_beit.py +++ b/src/transformers/models/beit/modeling_beit.py @@ -29,9 +29,9 @@ from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPooling, + ImageClassifierOutput, MaskedLMOutput, SemanticSegmenterOutput, - SequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( @@ -851,7 +851,7 @@ def __init__(self, config: BeitModel) -> None: @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, + output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC, expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, ) @@ -863,7 +863,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[tuple, SequenceClassifierOutput]: + ) -> Union[tuple, ImageClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., @@ -909,7 +909,7 @@ def forward( output = (logits,) + outputs[2:] return ((loss,) + output) if loss is not None else output - return SequenceClassifierOutput( + return ImageClassifierOutput( loss=loss, logits=logits, hidden_states=outputs.hidden_states, diff --git a/src/transformers/models/convnext/modeling_convnext.py b/src/transformers/models/convnext/modeling_convnext.py index b202c548d4df..a0ca01d4c8ac 100755 --- a/src/transformers/models/convnext/modeling_convnext.py +++ b/src/transformers/models/convnext/modeling_convnext.py @@ -14,8 +14,6 @@ # limitations under the License. """ PyTorch ConvNext model.""" -from dataclasses import dataclass -from typing import Optional, Tuple import torch import torch.utils.checkpoint @@ -23,14 +21,13 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, +from ...modeling_outputs import ( + BaseModelOutputWithNoAttention, + BaseModelOutputWithPoolingAndNoAttention, + ImageClassifierOutputWithNoAttention, ) +from ...modeling_utils import PreTrainedModel +from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_convnext import ConvNextConfig @@ -54,66 +51,6 @@ ] -@dataclass -class ConvNextEncoderOutput(ModelOutput): - """ - Class for [`ConvNextEncoder`]'s outputs, with potential hidden states (feature maps). - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Last hidden states (final feature map) of the last stage of the model. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of - shape `(batch_size, num_channels, height, width)`. Hidden-states (also called feature maps) of the model at - the output of each stage. - """ - - last_hidden_state: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - -@dataclass -class ConvNextModelOutput(ModelOutput): - """ - Class for [`ConvNextModel`]'s outputs, with potential hidden states (feature maps). - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Last hidden states (final feature map) of the last stage of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, config.dim[-1])`): - Global average pooling of the last feature map followed by a layernorm. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of - shape `(batch_size, num_channels, height, width)`. Hidden-states (also called feature maps) of the model at - the output of each stage. - """ - - last_hidden_state: torch.FloatTensor = None - pooler_output: Optional[torch.FloatTensor] = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - -@dataclass -class ConvNextClassifierOutput(ModelOutput): - """ - Class for [`ConvNextForImageClassification`]'s outputs, with potential hidden states (feature maps). - - Args: - loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): - Classification (or regression if config.num_labels==1) loss. - logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`): - Classification (or regression if config.num_labels==1) scores (before SoftMax). - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of - shape `(batch_size, num_channels, height, width)`. Hidden-states (also called feature maps) of the model at - the output of each stage. - """ - - loss: Optional[torch.FloatTensor] = None - logits: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - # Stochastic depth implementation # Taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/drop.py def drop_path(x, drop_prob: float = 0.0, training: bool = False): @@ -302,7 +239,7 @@ def forward(self, hidden_states, output_hidden_states=False, return_dict=True): if not return_dict: return tuple(v for v in [hidden_states, all_hidden_states] if v is not None) - return ConvNextEncoderOutput( + return BaseModelOutputWithNoAttention( last_hidden_state=hidden_states, hidden_states=all_hidden_states, ) @@ -383,7 +320,7 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, - output_type=ConvNextModelOutput, + output_type=BaseModelOutputWithPoolingAndNoAttention, config_class=_CONFIG_FOR_DOC, modality="vision", expected_output=_EXPECTED_OUTPUT_SHAPE, @@ -413,7 +350,7 @@ def forward(self, pixel_values=None, output_hidden_states=None, return_dict=None if not return_dict: return (last_hidden_state, pooled_output) + encoder_outputs[1:] - return ConvNextModelOutput( + return BaseModelOutputWithPoolingAndNoAttention( last_hidden_state=last_hidden_state, pooler_output=pooled_output, hidden_states=encoder_outputs.hidden_states, @@ -446,7 +383,7 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ConvNextClassifierOutput, + output_type=ImageClassifierOutputWithNoAttention, config_class=_CONFIG_FOR_DOC, expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, ) @@ -491,7 +428,7 @@ def forward(self, pixel_values=None, labels=None, output_hidden_states=None, ret output = (logits,) + outputs[2:] return ((loss,) + output) if loss is not None else output - return ConvNextClassifierOutput( + return ImageClassifierOutputWithNoAttention( loss=loss, logits=logits, hidden_states=outputs.hidden_states, diff --git a/src/transformers/models/deit/modeling_deit.py b/src/transformers/models/deit/modeling_deit.py index 012d535c2536..11591aeaeed5 100644 --- a/src/transformers/models/deit/modeling_deit.py +++ b/src/transformers/models/deit/modeling_deit.py @@ -26,7 +26,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, MaskedLMOutput, SequenceClassifierOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput, MaskedLMOutput from ...modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( ModelOutput, @@ -693,7 +693,7 @@ def __init__(self, config: DeiTConfig) -> None: self.post_init() @add_start_docstrings_to_model_forward(DEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @replace_return_docstrings(output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC) def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -702,7 +702,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[tuple, SequenceClassifierOutput]: + ) -> Union[tuple, ImageClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., @@ -777,7 +777,7 @@ def forward( output = (logits,) + outputs[2:] return ((loss,) + output) if loss is not None else output - return SequenceClassifierOutput( + return ImageClassifierOutput( loss=loss, logits=logits, hidden_states=outputs.hidden_states, diff --git a/src/transformers/models/poolformer/modeling_poolformer.py b/src/transformers/models/poolformer/modeling_poolformer.py index 6adf01315e34..2335c7cdc40c 100755 --- a/src/transformers/models/poolformer/modeling_poolformer.py +++ b/src/transformers/models/poolformer/modeling_poolformer.py @@ -16,7 +16,6 @@ import collections.abc -from dataclasses import dataclass from typing import Optional, Tuple, Union import torch @@ -25,14 +24,9 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN +from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, -) +from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_poolformer import PoolFormerConfig @@ -63,47 +57,6 @@ def to_2tuple(x): return (x, x) -@dataclass -class PoolFormerModelOutput(ModelOutput): - """ - Class for PoolFormerModel's outputs, with potential hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer - plus the initial embedding outputs. - """ - - last_hidden_state: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - -@dataclass -class PoolFormerClassifierOutput(ModelOutput): - """ - Class for PoolformerForImageClassification's outputs. - - Args: - loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): - Classification (or regression if config.num_labels==1) loss. - logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`): - Classification (or regression if config.num_labels==1) scores (before SoftMax). - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, num_channels, height, width)`. - - Hidden-states of the model at the output of each layer plus the initial embedding outputs. - """ - - loss: Optional[torch.FloatTensor] = None - logits: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - def drop_path(x, drop_prob: float = 0.0, training: bool = False): """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, the original name is @@ -295,7 +248,7 @@ def forward(self, pixel_values, output_hidden_states=False, return_dict=True): # Get patch embeddings from hidden_states hidden_states = embedding_layer(hidden_states) # Send the embeddings through the blocks - for i, blk in enumerate(block_layer): + for _, blk in enumerate(block_layer): layer_outputs = blk(hidden_states) hidden_states = layer_outputs[0] @@ -305,7 +258,7 @@ def forward(self, pixel_values, output_hidden_states=False, return_dict=True): if not return_dict: return tuple(v for v in [hidden_states, all_hidden_states] if v is not None) - return PoolFormerModelOutput(last_hidden_state=hidden_states, hidden_states=all_hidden_states) + return BaseModelOutputWithNoAttention(last_hidden_state=hidden_states, hidden_states=all_hidden_states) class PoolFormerPreTrainedModel(PreTrainedModel): @@ -374,7 +327,7 @@ def get_input_embeddings(self): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, - output_type=PoolFormerModelOutput, + output_type=BaseModelOutputWithNoAttention, config_class=_CONFIG_FOR_DOC, modality="vision", expected_output=_EXPECTED_OUTPUT_SHAPE, @@ -384,7 +337,7 @@ def forward( pixel_values: Optional[torch.FloatTensor] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, PoolFormerModelOutput]: + ) -> Union[Tuple, BaseModelOutputWithNoAttention]: output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -403,7 +356,7 @@ def forward( if not return_dict: return (sequence_output, None) + encoder_outputs[1:] - return PoolFormerModelOutput( + return BaseModelOutputWithNoAttention( last_hidden_state=sequence_output, hidden_states=encoder_outputs.hidden_states, ) @@ -445,7 +398,7 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=PoolFormerClassifierOutput, + output_type=ImageClassifierOutputWithNoAttention, config_class=_CONFIG_FOR_DOC, expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, ) @@ -455,7 +408,7 @@ def forward( labels: Optional[torch.LongTensor] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, PoolFormerClassifierOutput]: + ) -> Union[Tuple, ImageClassifierOutputWithNoAttention]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., @@ -501,4 +454,4 @@ def forward( output = (logits,) + outputs[2:] return ((loss,) + output) if loss is not None else output - return PoolFormerClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) + return ImageClassifierOutputWithNoAttention(loss=loss, logits=logits, hidden_states=outputs.hidden_states) diff --git a/src/transformers/models/resnet/modeling_resnet.py b/src/transformers/models/resnet/modeling_resnet.py index 7e74cdf8dcb7..00c589d68f7b 100644 --- a/src/transformers/models/resnet/modeling_resnet.py +++ b/src/transformers/models/resnet/modeling_resnet.py @@ -14,8 +14,7 @@ # limitations under the License. """ PyTorch ResNet model.""" -from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional import torch import torch.utils.checkpoint @@ -23,7 +22,11 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import ImageClassifierOutput, ModelOutput +from ...modeling_outputs import ( + BaseModelOutputWithNoAttention, + BaseModelOutputWithPoolingAndNoAttention, + ImageClassifierOutputWithNoAttention, +) from ...modeling_utils import PreTrainedModel from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_resnet import ResNetConfig @@ -49,47 +52,6 @@ ] -@dataclass -class ResNetEncoderOutput(ModelOutput): - """ - ResNet encoder's output, with potential hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Sequence of hidden-states at the output of the last layer of the model. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, num_channels, height, width)`. - - Hidden-states of the model at the output of each layer plus the initial embedding outputs. - """ - - last_hidden_state: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - -@dataclass -class ResNetModelOutput(ModelOutput): - """ - ResNet model's output, with potential hidden states. - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Sequence of hidden-states at the output of the last layer of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, config.hidden_sizes[-1])`): - The pooled last hidden state. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of - shape `(batch_size, num_channels, height, width)`. - - Hidden-states of the model at the output of each layer plus the initial embedding outputs. - """ - - last_hidden_state: torch.FloatTensor = None - pooler_output: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - class ResNetConvLayer(nn.Sequential): def __init__( self, in_channels: int, out_channels: int, kernel_size: int = 3, stride: int = 1, activation: str = "relu" @@ -228,7 +190,7 @@ def __init__(self, config: ResNetConfig): def forward( self, hidden_state: Tensor, output_hidden_states: bool = False, return_dict: bool = True - ) -> ResNetEncoderOutput: + ) -> BaseModelOutputWithNoAttention: hidden_states = () if output_hidden_states else None for stage_module in self.stages: @@ -243,7 +205,7 @@ def forward( if not return_dict: return tuple(v for v in [hidden_state, hidden_states] if v is not None) - return ResNetEncoderOutput( + return BaseModelOutputWithNoAttention( last_hidden_state=hidden_state, hidden_states=hidden_states, ) @@ -315,14 +277,14 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, - output_type=ResNetModelOutput, + output_type=BaseModelOutputWithPoolingAndNoAttention, config_class=_CONFIG_FOR_DOC, modality="vision", expected_output=_EXPECTED_OUTPUT_SHAPE, ) def forward( self, pixel_values: Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None - ) -> ResNetModelOutput: + ) -> BaseModelOutputWithPoolingAndNoAttention: output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -341,7 +303,7 @@ def forward( if not return_dict: return (last_hidden_state, pooled_output) + encoder_outputs[1:] - return ResNetModelOutput( + return BaseModelOutputWithPoolingAndNoAttention( last_hidden_state=last_hidden_state, pooler_output=pooled_output, hidden_states=encoder_outputs.hidden_states, @@ -372,7 +334,7 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=ImageClassifierOutput, + output_type=ImageClassifierOutputWithNoAttention, config_class=_CONFIG_FOR_DOC, expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, ) @@ -382,7 +344,7 @@ def forward( labels: Tensor = None, output_hidden_states: bool = None, return_dict: bool = None, - ) -> ImageClassifierOutput: + ) -> ImageClassifierOutputWithNoAttention: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., @@ -423,4 +385,4 @@ def forward( output = (logits,) + outputs[2:] return (loss,) + output if loss is not None else output - return ImageClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) + return ImageClassifierOutputWithNoAttention(loss=loss, logits=logits, hidden_states=outputs.hidden_states) diff --git a/src/transformers/models/segformer/modeling_segformer.py b/src/transformers/models/segformer/modeling_segformer.py index d180f4232b9e..d90e285ebcb2 100755 --- a/src/transformers/models/segformer/modeling_segformer.py +++ b/src/transformers/models/segformer/modeling_segformer.py @@ -24,7 +24,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, SemanticSegmenterOutput, SequenceClassifierOutput +from ...modeling_outputs import BaseModelOutput, ImageClassifierOutput, SemanticSegmenterOutput from ...modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( add_code_sample_docstrings, @@ -57,6 +57,33 @@ ] +class SegFormerImageClassifierOutput(ImageClassifierOutput): + """ + Base class for outputs of image classification models. + + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Classification (or regression if config.num_labels==1) loss. + logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`): + Classification (or regression if config.num_labels==1) scores (before SoftMax). + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each stage) of shape `(batch_size, num_channels, height, width)`. Hidden-states (also + called feature maps) of the model at the output of each stage. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + loss: Optional[torch.FloatTensor] = None + logits: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + # Copied from transformers.models.convnext.modeling_convnext.drop_path def drop_path(x, drop_prob: float = 0.0, training: bool = False, scale_by_keep=True): """ @@ -558,7 +585,7 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, + output_type=SegFormerImageClassifierOutput, config_class=_CONFIG_FOR_DOC, expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, ) @@ -569,7 +596,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, SequenceClassifierOutput]: + ) -> Union[Tuple, SegFormerImageClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., @@ -625,7 +652,7 @@ def forward( output = (logits,) + outputs[1:] return ((loss,) + output) if loss is not None else output - return SequenceClassifierOutput( + return SegFormerImageClassifierOutput( loss=loss, logits=logits, hidden_states=outputs.hidden_states, diff --git a/src/transformers/models/van/modeling_van.py b/src/transformers/models/van/modeling_van.py index 84c39b40af4e..3ea59cbae8d3 100644 --- a/src/transformers/models/van/modeling_van.py +++ b/src/transformers/models/van/modeling_van.py @@ -16,8 +16,6 @@ import math from collections import OrderedDict -from dataclasses import dataclass -from typing import Optional, Tuple import torch import torch.utils.checkpoint @@ -25,14 +23,13 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_utils import PreTrainedModel -from ...utils import ( - ModelOutput, - add_code_sample_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, +from ...modeling_outputs import ( + BaseModelOutputWithNoAttention, + BaseModelOutputWithPoolingAndNoAttention, + ImageClassifierOutputWithNoAttention, ) +from ...modeling_utils import PreTrainedModel +from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_van import VanConfig @@ -56,63 +53,6 @@ ] -@dataclass -class VanEncoderOutput(ModelOutput): - """ - Class for [`VanEncoder`]'s outputs, with potential hidden states (feature maps). - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Last hidden states (final feature map) of the last stage of the model. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, num_channels, - height, width)`. Hidden-states (also called feature maps) of the model at the output of each stage. - """ - - last_hidden_state: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - -@dataclass -class VanModelOutput(ModelOutput): - """ - Class for [`VanModel`]'s outputs, with potential hidden states (feature maps). - - Args: - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Last hidden states (final feature map) of the last stage of the model. - pooler_output (`torch.FloatTensor` of shape `(batch_size, config.hidden_sizes[-1])`): - Global average pooling of the last feature map followed by a layernorm. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, num_channels, - height, width)`. Hidden-states (also called feature maps) of the model at the output of each stage. - """ - - last_hidden_state: torch.FloatTensor = None - pooler_output: Optional[torch.FloatTensor] = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - -@dataclass -class VanClassifierOutput(ModelOutput): - """ - Class for [`VanForImageClassification`]'s outputs, with potential hidden states (feature maps). - - Args: - loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): - Classification (or regression if config.num_labels==1) loss. - logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`): - Classification (or regression if config.num_labels==1) scores (before SoftMax). - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, num_channels, - height, width)`. Hidden-states (also called feature maps) of the model at the output of each stage. - """ - - loss: Optional[torch.FloatTensor] = None - logits: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - - # Stochastic depth implementation # Taken from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/drop.py def drop_path(x, drop_prob: float = 0.0, training: bool = False): @@ -388,7 +328,7 @@ def forward(self, hidden_state, output_hidden_states=False, return_dict=True): if not return_dict: return tuple(v for v in [hidden_state, all_hidden_states] if v is not None) - return VanEncoderOutput(last_hidden_state=hidden_state, hidden_states=all_hidden_states) + return BaseModelOutputWithNoAttention(last_hidden_state=hidden_state, hidden_states=all_hidden_states) class VanPreTrainedModel(PreTrainedModel): @@ -466,7 +406,7 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, - output_type=VanModelOutput, + output_type=BaseModelOutputWithPoolingAndNoAttention, config_class=_CONFIG_FOR_DOC, modality="vision", expected_output=_EXPECTED_OUTPUT_SHAPE, @@ -489,7 +429,7 @@ def forward(self, pixel_values, output_hidden_states=None, return_dict=None): if not return_dict: return (last_hidden_state, pooled_output) + encoder_outputs[1:] - return VanModelOutput( + return BaseModelOutputWithPoolingAndNoAttention( last_hidden_state=last_hidden_state, pooler_output=pooled_output, hidden_states=encoder_outputs.hidden_states, @@ -519,7 +459,7 @@ def __init__(self, config): @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=VanClassifierOutput, + output_type=ImageClassifierOutputWithNoAttention, config_class=_CONFIG_FOR_DOC, expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, ) @@ -565,4 +505,4 @@ def forward(self, pixel_values=None, labels=None, output_hidden_states=None, ret output = (logits,) + outputs[2:] return ((loss,) + output) if loss is not None else output - return VanClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states) + return ImageClassifierOutputWithNoAttention(loss=loss, logits=logits, hidden_states=outputs.hidden_states) diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index 9679fef670b2..ef4a4191ccf8 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -25,7 +25,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, MaskedLMOutput, SequenceClassifierOutput +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput, MaskedLMOutput from ...modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( add_code_sample_docstrings, @@ -740,7 +740,7 @@ def __init__(self, config: ViTConfig) -> None: @add_code_sample_docstrings( processor_class=_FEAT_EXTRACTOR_FOR_DOC, checkpoint=_IMAGE_CLASS_CHECKPOINT, - output_type=SequenceClassifierOutput, + output_type=ImageClassifierOutput, config_class=_CONFIG_FOR_DOC, expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, ) @@ -753,7 +753,7 @@ def forward( output_hidden_states: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[tuple, SequenceClassifierOutput]: + ) -> Union[tuple, ImageClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., @@ -801,7 +801,7 @@ def forward( output = (logits,) + outputs[2:] return ((loss,) + output) if loss is not None else output - return SequenceClassifierOutput( + return ImageClassifierOutput( loss=loss, logits=logits, hidden_states=outputs.hidden_states, From 99a01423b94d72265237698edd28cda72a94db65 Mon Sep 17 00:00:00 2001 From: chenbohua3 Date: Thu, 31 Mar 2022 19:39:46 +0800 Subject: [PATCH 32/54] make tuple annotation more specific to avoid failures during symbolic_trace (#16490) * make tuple annotation more specific to avoid failures during symbolic_trace * make tuple annotation more specific to avoid failures during symbolic_trace --- src/transformers/models/bert/modeling_bert.py | 26 +++++++++---------- .../models/data2vec/modeling_data2vec_text.py | 10 +++---- .../models/electra/modeling_electra.py | 8 +++--- .../models/layoutlm/modeling_layoutlm.py | 8 +++--- .../megatron_bert/modeling_megatron_bert.py | 2 +- .../models/mobilebert/modeling_mobilebert.py | 8 +++--- .../models/realm/modeling_realm.py | 8 +++--- .../models/rembert/modeling_rembert.py | 4 +-- .../models/roberta/modeling_roberta.py | 10 +++---- .../models/splinter/modeling_splinter.py | 8 +++--- .../models/tapas/modeling_tapas.py | 4 +-- .../xlm_roberta_xl/modeling_xlm_roberta_xl.py | 4 +-- 12 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/transformers/models/bert/modeling_bert.py b/src/transformers/models/bert/modeling_bert.py index 88c99c8a2ee4..6e1a41a6b3bd 100755 --- a/src/transformers/models/bert/modeling_bert.py +++ b/src/transformers/models/bert/modeling_bert.py @@ -264,7 +264,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -398,7 +398,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -466,7 +466,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( @@ -546,7 +546,7 @@ def forward( output_attentions: Optional[bool] = False, output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None @@ -906,7 +906,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if @@ -1059,7 +1059,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BertForPreTrainingOutput]: + ) -> Union[Tuple[torch.Tensor], BertForPreTrainingOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., @@ -1173,7 +1173,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention @@ -1330,7 +1330,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, MaskedLMOutput]: + ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., @@ -1419,7 +1419,7 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, **kwargs, - ) -> Union[Tuple, NextSentencePredictorOutput]: + ) -> Union[Tuple[torch.Tensor], NextSentencePredictorOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair @@ -1533,7 +1533,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, SequenceClassifierOutput]: + ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1633,7 +1633,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, MultipleChoiceModelOutput]: + ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., @@ -1732,7 +1732,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, TokenClassifierOutput]: + ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. @@ -1814,7 +1814,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, QuestionAnsweringModelOutput]: + ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. diff --git a/src/transformers/models/data2vec/modeling_data2vec_text.py b/src/transformers/models/data2vec/modeling_data2vec_text.py index 87a2e810e298..13988eafb238 100644 --- a/src/transformers/models/data2vec/modeling_data2vec_text.py +++ b/src/transformers/models/data2vec/modeling_data2vec_text.py @@ -200,7 +200,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -336,7 +336,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -407,7 +407,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( @@ -488,7 +488,7 @@ def forward( output_attentions: Optional[bool] = False, output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None @@ -764,7 +764,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if diff --git a/src/transformers/models/electra/modeling_electra.py b/src/transformers/models/electra/modeling_electra.py index ce0b9c36cd25..e5347cc19c24 100644 --- a/src/transformers/models/electra/modeling_electra.py +++ b/src/transformers/models/electra/modeling_electra.py @@ -257,7 +257,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -393,7 +393,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -464,7 +464,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( @@ -545,7 +545,7 @@ def forward( output_attentions: Optional[bool] = False, output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None diff --git a/src/transformers/models/layoutlm/modeling_layoutlm.py b/src/transformers/models/layoutlm/modeling_layoutlm.py index 2eea904360c6..2b6a8dca7bc4 100644 --- a/src/transformers/models/layoutlm/modeling_layoutlm.py +++ b/src/transformers/models/layoutlm/modeling_layoutlm.py @@ -172,7 +172,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -308,7 +308,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -379,7 +379,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( @@ -460,7 +460,7 @@ def forward( output_attentions: Optional[bool] = False, output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None diff --git a/src/transformers/models/megatron_bert/modeling_megatron_bert.py b/src/transformers/models/megatron_bert/modeling_megatron_bert.py index 36b7e3dfdf2a..9cdea376c74f 100755 --- a/src/transformers/models/megatron_bert/modeling_megatron_bert.py +++ b/src/transformers/models/megatron_bert/modeling_megatron_bert.py @@ -235,7 +235,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys diff --git a/src/transformers/models/mobilebert/modeling_mobilebert.py b/src/transformers/models/mobilebert/modeling_mobilebert.py index ff971110e140..d57b83570006 100644 --- a/src/transformers/models/mobilebert/modeling_mobilebert.py +++ b/src/transformers/models/mobilebert/modeling_mobilebert.py @@ -1245,7 +1245,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, SequenceClassifierOutput]: + ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1347,7 +1347,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, QuestionAnsweringModelOutput]: + ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: r""" start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. @@ -1452,7 +1452,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, MultipleChoiceModelOutput]: + ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., @@ -1552,7 +1552,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, TokenClassifierOutput]: + ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. diff --git a/src/transformers/models/realm/modeling_realm.py b/src/transformers/models/realm/modeling_realm.py index 59e1f275bc36..fbaa1c14b7f0 100644 --- a/src/transformers/models/realm/modeling_realm.py +++ b/src/transformers/models/realm/modeling_realm.py @@ -271,7 +271,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -407,7 +407,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -478,7 +478,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( @@ -559,7 +559,7 @@ def forward( output_attentions: Optional[bool] = False, output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None diff --git a/src/transformers/models/rembert/modeling_rembert.py b/src/transformers/models/rembert/modeling_rembert.py index 428cdd5ad98a..76b4c715d758 100755 --- a/src/transformers/models/rembert/modeling_rembert.py +++ b/src/transformers/models/rembert/modeling_rembert.py @@ -370,7 +370,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -441,7 +441,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( diff --git a/src/transformers/models/roberta/modeling_roberta.py b/src/transformers/models/roberta/modeling_roberta.py index ca6678282e5c..520e24034e69 100644 --- a/src/transformers/models/roberta/modeling_roberta.py +++ b/src/transformers/models/roberta/modeling_roberta.py @@ -200,7 +200,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -336,7 +336,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -407,7 +407,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( @@ -488,7 +488,7 @@ def forward( output_attentions: Optional[bool] = False, output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None @@ -761,7 +761,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if diff --git a/src/transformers/models/splinter/modeling_splinter.py b/src/transformers/models/splinter/modeling_splinter.py index 64ac946d9b87..86e1c846c82f 100755 --- a/src/transformers/models/splinter/modeling_splinter.py +++ b/src/transformers/models/splinter/modeling_splinter.py @@ -144,7 +144,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -280,7 +280,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -351,7 +351,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( @@ -432,7 +432,7 @@ def forward( output_attentions: Optional[bool] = False, output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py index 5b3fc4b46031..eaf4fc8435b1 100644 --- a/src/transformers/models/tapas/modeling_tapas.py +++ b/src/transformers/models/tapas/modeling_tapas.py @@ -492,7 +492,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: self_outputs = self.self( hidden_states, attention_mask, @@ -563,7 +563,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None self_attention_outputs = self.attention( diff --git a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py index aa7e47cc8806..04d30b432095 100644 --- a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py +++ b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py @@ -194,7 +194,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, output_attentions: Optional[bool] = False, - ) -> Tuple: + ) -> Tuple[torch.Tensor]: mixed_query_layer = self.query(hidden_states) # If this is instantiated as a cross-attention module, the keys @@ -732,7 +732,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPoolingAndCrossAttentions]: + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: r""" encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if From 98939e6aee54dc36451f2695d6501f8c7c986381 Mon Sep 17 00:00:00 2001 From: Juanjo do Olmo <87780148+SimplyJuanjo@users.noreply.github.com> Date: Thu, 31 Mar 2022 13:43:31 +0200 Subject: [PATCH 33/54] Spanish translation of the file multilingual.mdx (#16329) * Duplication of the source eng file * Spanish translation of the file multilingual.mdx * Update docs/source_es/multilingual.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/multilingual.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/multilingual.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/multilingual.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/multilingual.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/multilingual.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/multilingual.mdx Co-authored-by: Omar U. Espejel * Fix nits and finish translation Co-authored-by: Omar U. Espejel --- docs/source_es/multilingual.mdx | 175 ++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 docs/source_es/multilingual.mdx diff --git a/docs/source_es/multilingual.mdx b/docs/source_es/multilingual.mdx new file mode 100644 index 000000000000..4849416a44db --- /dev/null +++ b/docs/source_es/multilingual.mdx @@ -0,0 +1,175 @@ + + +# Modelos multilingües para inferencia + +[[open-in-colab]] + +Existen varios modelos multilingües en 🤗 Transformers y su uso para inferencia difiere de los modelos monolingües. Sin embargo, no *todos* los usos de los modelos multilingües son diferentes. Algunos modelos, como [bert-base-multilingual-uncased](https://huggingface.co/bert-base-multilingual-uncased), pueden utilizarse igual que un modelo monolingüe. Esta guía te enseñará cómo utilizar modelos multilingües cuyo uso difiere en la inferencia. + +## XLM + +XLM tiene diez checkpoints diferentes de los cuales solo uno es monolingüe. Los nueve checkpoints restantes del modelo pueden dividirse en dos categorías: los checkpoints que utilizan language embeddings y los que no. + +### XLM con language embeddings + +Los siguientes modelos XLM usan language embeddings para especificar el lenguaje utilizado en la inferencia: + +- `xlm-mlm-ende-1024` (Masked language modeling, English-German) +- `xlm-mlm-enfr-1024` (Masked language modeling, English-French) +- `xlm-mlm-enro-1024` (Masked language modeling, English-Romanian) +- `xlm-mlm-xnli15-1024` (Masked language modeling, XNLI languages) +- `xlm-mlm-tlm-xnli15-1024` (Masked language modeling + translation, XNLI languages) +- `xlm-clm-enfr-1024` (Causal language modeling, English-French) +- `xlm-clm-ende-1024` (Causal language modeling, English-German) + +Los language embeddings son representados como un tensor de la mismas dimensiones que los `input_ids` pasados al modelo. Los valores de estos tensores dependen del idioma utilizado y se identifican mediante los atributos `lang2id` y `id2lang` del tokenizador. + +En este ejemplo, carga el checkpoint `xlm-clm-enfr-1024` (Causal language modeling, English-French): + +```py +>>> import torch +>>> from transformers import XLMTokenizer, XLMWithLMHeadModel + +>>> tokenizer = XLMTokenizer.from_pretrained("xlm-clm-enfr-1024") +>>> model = XLMWithLMHeadModel.from_pretrained("xlm-clm-enfr-1024") +``` + +El atributo `lang2id` del tokenizador muestra los idiomas de este modelo y sus ids: + +```py +>>> print(tokenizer.lang2id) +{'en': 0, 'fr': 1} +``` + +A continuación, crea un input de ejemplo: + +```py +>>> input_ids = torch.tensor([tokenizer.encode("Wikipedia was used to")]) # batch size of 1 +``` + +Establece el id del idioma, por ejemplo `"en"`, y utilízalo para definir el language embedding. El language embedding es un tensor lleno de `0` ya que es el id del idioma para inglés. Este tensor debe ser del mismo tamaño que `input_ids`. + +```py +>>> language_id = tokenizer.lang2id["en"] # 0 +>>> langs = torch.tensor([language_id] * input_ids.shape[1]) # torch.tensor([0, 0, 0, ..., 0]) + +>>> # We reshape it to be of size (batch_size, sequence_length) +>>> langs = langs.view(1, -1) # is now of shape [1, sequence_length] (we have a batch size of 1) +``` + +Ahora puedes pasar los `input_ids` y el language embedding al modelo: + +```py +>>> outputs = model(input_ids, langs=langs) +``` + +El script [run_generation.py](https://github.com/huggingface/transformers/tree/master/examples/pytorch/text-generation/run_generation.py) puede generar texto con language embeddings utilizando los checkpoints `xlm-clm`. + +### XLM sin language embeddings + +Los siguientes modelos XLM no requieren language embeddings durante la inferencia: + +- `xlm-mlm-17-1280` (modelado de lenguaje enmascarado, 17 idiomas) +- `xlm-mlm-100-1280` (modelado de lenguaje enmascarado, 100 idiomas) + +Estos modelos se utilizan para representaciones genéricas de frases a diferencia de los anteriores checkpoints XLM. + +## BERT + +Los siguientes modelos de BERT pueden utilizarse para tareas multilingües: + +- `bert-base-multilingual-uncased` (modelado de lenguaje enmascarado + predicción de la siguiente oración, 102 idiomas) +- `bert-base-multilingual-cased` (modelado de lenguaje enmascarado + predicción de la siguiente oración, 104 idiomas) + +Estos modelos no requieren language embeddings durante la inferencia. Deben identificar la lengua a partir del +contexto e inferir en consecuencia. + +## XLM-RoBERTa + +Los siguientes modelos de XLM-RoBERTa pueden utilizarse para tareas multilingües: + +- `xlm-roberta-base` (modelado de lenguaje enmascarado, 100 idiomas) +- `xlm-roberta-large` (Modelado de lenguaje enmascarado, 100 idiomas) + +XLM-RoBERTa se entrenó con 2,5 TB de datos CommonCrawl recién creados y depurados en 100 idiomas. Proporciona fuertes ventajas sobre los modelos multilingües publicados anteriormente como mBERT o XLM en tareas posteriores como la clasificación, el etiquetado de secuencias y la respuesta a preguntas. + +## M2M100 + +Los siguientes modelos de M2M100 pueden utilizarse para traducción multilingüe: + +- `facebook/m2m100_418M` (traducción) +- `facebook/m2m100_1.2B` (traducción) + +En este ejemplo, carga el checkpoint `facebook/m2m100_418M` para traducir del chino al inglés. Puedes establecer el idioma de origen en el tokenizador: + +```py +>>> from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer + +>>> en_text = "Do not meddle in the affairs of wizards, for they are subtle and quick to anger." +>>> chinese_text = "不要插手巫師的事務, 因為他們是微妙的, 很快就會發怒." + +>>> tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M", src_lang="zh") +>>> model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") +``` + +Tokeniza el texto: + +```py +>>> encoded_zh = tokenizer(chinese_text, return_tensors="pt") +``` + +M2M100 fuerza el id del idioma de destino como el primer token generado para traducir al idioma de destino.. Establece el `forced_bos_token_id` a `en` en el método `generate` para traducir al inglés: + +```py +>>> generated_tokens = model.generate(**encoded_zh, forced_bos_token_id=tokenizer.get_lang_id("en")) +>>> tokenizer.batch_decode(generated_tokens, skip_special_tokens=True) +'Do not interfere with the matters of the witches, because they are delicate and will soon be angry.' +``` + +## MBart + +Los siguientes modelos de MBart pueden utilizarse para traducción multilingüe: + +- `facebook/mbart-large-50-one-to-many-mmt` (traducción automática multilingüe de uno a muchos, 50 idiomas) +- `facebook/mbart-large-50-many-to-many-mmt` (traducción automática multilingüe de muchos a muchos, 50 idiomas) +- `facebook/mbart-large-50-many-to-one-mmt` (traducción automática multilingüe muchos a uno, 50 idiomas) +- `facebook/mbart-large-50` (traducción multilingüe, 50 idiomas) +- `facebook/mbart-large-cc25` + +En este ejemplo, carga el checkpoint `facebook/mbart-large-50-many-to-many-mmt` para traducir del finlandés al inglés. Puedes establecer el idioma de origen en el tokenizador: + +```py +>>> from transformers import AutoTokenizer, AutoModelForSeq2SeqLM + +>>> en_text = "Do not meddle in the affairs of wizards, for they are subtle and quick to anger." +>>> fi_text = "Älä sekaannu velhojen asioihin, sillä ne ovat hienovaraisia ja nopeasti vihaisia." + +>>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-50-many-to-many-mmt", src_lang="fi_FI") +>>> model = AutoModelForSeq2SeqLM.from_pretrained("facebook/mbart-large-50-many-to-many-mmt") +``` + +Tokeniza el texto: + +```py +>>> encoded_en = tokenizer(en_text, return_tensors="pt") +``` + +MBart fuerza el id del idioma de destino como el primer token generado para traducirlo. Establece el `forced_bos_token_id` a `en` en el método `generate` para traducir al inglés: + +```py +>>> generated_tokens = model.generate(**encoded_en, forced_bos_token_id=tokenizer.lang_code_to_id("en_XX")) +>>> tokenizer.batch_decode(generated_tokens, skip_special_tokens=True) +"Don't interfere with the wizard's affairs, because they are subtle, will soon get angry." +``` + +Si estás usando el checkpoint `facebook/mbart-large-50-many-to-one-mmt` no necesitas forzar el id del idioma de destino como el primer token generado, de lo contrario el uso es el mismo. From c551addeb09a9d88b7126b935b774c6b730c6dbd Mon Sep 17 00:00:00 2001 From: Liliana Badillo Date: Thu, 31 Mar 2022 05:44:47 -0600 Subject: [PATCH 34/54] Translate installation.mdx to Spanish (#16229) * Translate installation.mdx to Spanish * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Update docs/source_es/installation.mdx Co-authored-by: Omar U. Espejel * Fix nits and finish translation Co-authored-by: Omar U. Espejel --- docs/source_es/installation.mdx | 244 ++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 docs/source_es/installation.mdx diff --git a/docs/source_es/installation.mdx b/docs/source_es/installation.mdx new file mode 100644 index 000000000000..1e0b587e283b --- /dev/null +++ b/docs/source_es/installation.mdx @@ -0,0 +1,244 @@ + + +# Guía de instalación + +En esta guía puedes encontrar información para instalar 🤗 Transformers para cualquier biblioteca de Machine Learning con la que estés trabajando. Además, encontrarás información sobre cómo establecer el caché y cómo configurar 🤗 Transformers para correrlo de manera offline (opcional). + +🤗 Transformers ha sido probada en Python 3.6+, PyTorch 1.1.0+, TensorFlow 2.0+, y Flax. Para instalar la biblioteca de deep learning con la que desees trabajar, sigue las instrucciones correspondientes listadas a continuación: + +* [PyTorch](https://pytorch.org/get-started/locally/) +* [TensorFlow 2.0](https://www.tensorflow.org/install/pip) +* [Flax](https://flax.readthedocs.io/en/latest/) + +## Instalación con pip + +Es necesario instalar 🤗 Transformers en un [entorno virtual](https://docs.python.org/3/library/venv.html). Si necesitas más información sobre entornos virtuales de Python, consulta esta [guía](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/ +). Un entorno virtual facilita el manejo de proyectos y evita problemas de compatibilidad entre dependencias. + +Comienza por crear un entorno virtual en el directorio de tu proyecto: + +```bash +python -m venv .env +``` + +Activa el entorno virtual: + +```bash +source .env/bin/activate +``` + +Ahora puedes instalar 🤗 Transformers con el siguiente comando: + +```bash +pip install transformers +``` + +Solo para CPU, puedes instalar 🤗 Transformers y una biblioteca de deep learning con un comando de una sola línea. + +Por ejemplo, instala 🤗 Transformers y Pytorch: + +```bash +pip install transformers[torch] +``` + +🤗 Transformers y TensorFlow 2.0: + +```bash +pip install transformers[tf-cpu] +``` + +🤗 Transformers y Flax: + +```bash +pip install transformers[flax] +``` + +Por último, revisa si 🤗 Transformers ha sido instalada exitosamente con el siguiente comando que descarga un modelo pre-entrenado: + +```bash +python -c "from transformers import pipeline; print(pipeline('sentiment-analysis')('we love you'))" +``` +Después imprime la etiqueta y el puntaje: + +```bash +[{'label': 'POSITIVE', 'score': 0.9998704791069031}] +``` + +## Instalación desde la fuente + +Instala 🤗 Transformers desde la fuente con el siguiente comando: + +```bash +pip install git+https://github.com/huggingface/transformers +``` + +El comando de arriba instala la versión `master` más actual en vez de la última versión estable. La versión `master` es útil para obtener los últimos avances de 🤗 Transformers. Por ejemplo, se puede dar el caso de que un error fue corregido después de la última versión estable pero aún no se ha liberado un nuevo lanzamiento. Sin embargo, existe la posibilidad de que la versión `master` no sea estable. El equipo trata de mantener la versión `master` operacional y la mayoría de los errores son resueltos en unas cuantas horas o un día. Si encuentras algún problema, por favor abre un [Issue](https://github.com/huggingface/transformers/issues) para que pueda ser corregido más rápido. + +Verifica si 🤗 Transformers está instalada apropiadamente con el siguiente comando: + +```bash +python -c "from transformers import pipeline; print(pipeline('sentiment-analysis')('I love you'))" +``` + +## Instalación editable + +Necesitarás una instalación editable si deseas: +* Usar la versión `master` del código fuente. +* Contribuir a 🤗 Transformers y necesitas probar cambios en el código. + +Clona el repositorio e instala 🤗 Transformers con los siguientes comandos: + +```bash +git clone https://github.com/huggingface/transformers.git +cd transformers +pip install -e . +``` + +Éstos comandos van a ligar el directorio desde donde clonamos el repositorio al path de las bibliotecas de Python. Python ahora buscará dentro de la carpeta que clonaste además de los paths normales de la biblioteca. Por ejemplo, si los paquetes de Python se encuentran instalados en `~/anaconda3/envs/main/lib/python3.7/site-packages/`, Python también buscará en el directorio desde donde clonamos el repositorio `~/transformers/`. + + + +Debes mantener el directorio `transformers` si deseas seguir usando la biblioteca. + + + +Puedes actualizar tu copia local a la última versión de 🤗 Transformers con el siguiente comando: + +```bash +cd ~/transformers/ +git pull +``` + +El entorno de Python que creaste para la instalación de 🤗 Transformers encontrará la versión `master` en la siguiente ejecución. + +## Instalación con conda + +Puedes instalar 🤗 Transformers desde el canal de conda `huggingface` con el siguiente comando: + +```bash +conda install -c huggingface transformers +``` + +## Configuración de Caché + +Los modelos preentrenados se descargan y almacenan en caché localmente en: `~/.cache/huggingface/transformers/`. Este es el directorio predeterminado proporcionado por la variable de entorno de shell `TRANSFORMERS_CACHE`. En Windows, el directorio predeterminado es dado por `C:\Users\username\.cache\huggingface\transformers`. Puedes cambiar las variables de entorno de shell que se muestran a continuación, en orden de prioridad, para especificar un directorio de caché diferente: + +1. Variable de entorno del shell (por defecto): `TRANSFORMERS_CACHE`. +2. Variable de entorno del shell:`HF_HOME` + `transformers/`. +3. Variable de entorno del shell: `XDG_CACHE_HOME` + `/huggingface/transformers`. + + + +🤗 Transformers usará las variables de entorno de shell `PYTORCH_TRANSFORMERS_CACHE` o `PYTORCH_PRETRAINED_BERT_CACHE` si viene de una iteración anterior de la biblioteca y ha configurado esas variables de entorno, a menos que especifiques la variable de entorno de shell `TRANSFORMERS_CACHE`. + + + + +## Modo Offline + +🤗 Transformers puede ejecutarse en un entorno con firewall o fuera de línea (offline) usando solo archivos locales. Configura la variable de entorno `TRANSFORMERS_OFFLINE=1` para habilitar este comportamiento. + + + +Puedes añadir [🤗 Datasets](https://huggingface.co/docs/datasets/) al flujo de entrenamiento offline declarando la variable de entorno `HF_DATASETS_OFFLINE=1`. + + + +Por ejemplo, normalmente ejecutarías un programa en una red normal con firewall para instancias externas con el siguiente comando: + +```bash +python examples/pytorch/translation/run_translation.py --model_name_or_path t5-small --dataset_name wmt16 --dataset_config ro-en ... +``` + +Ejecuta este mismo programa en una instancia offline con el siguiente comando: + +```bash +HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ +python examples/pytorch/translation/run_translation.py --model_name_or_path t5-small --dataset_name wmt16 --dataset_config ro-en ... +``` + +El script ahora debería ejecutarse sin bloquearse ni esperar a que se agote el tiempo de espera porque sabe que solo debe buscar archivos locales. + +### Obtener modelos y tokenizers para uso offline + +Otra opción para usar 🤗 Transformers offline es descargando previamente los archivos y después apuntar al path local donde se encuentren. Hay tres maneras de hacer esto: + +* Descarga un archivo mediante la interfaz de usuario del [Model Hub](https://huggingface.co/models) haciendo click en el ícono ↓. + + ![download-icon](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/download-icon.png) + + +* Utiliza el flujo de [`PreTrainedModel.from_pretrained`] y [`PreTrainedModel.save_pretrained`]: + 1. Descarga previamente los archivos con [`PreTrainedModel.from_pretrained`]: + + ```py + >>> from transformers import AutoTokenizer, AutoModelForSeq2SeqLM + + >>> tokenizer = AutoTokenizer.from_pretrained("bigscience/T0_3B") + >>> model = AutoModelForSeq2SeqLM.from_pretrained("bigscience/T0_3B") + ``` + + + 2. Guarda los archivos en un directorio específico con [`PreTrainedModel.save_pretrained`]: + + ```py + >>> tokenizer.save_pretrained("./your/path/bigscience_t0") + >>> model.save_pretrained("./your/path/bigscience_t0") + ``` + + 3. Cuando te encuentres offline, recarga los archivos con [`PreTrainedModel.from_pretrained`] desde el directorio especificado: + + ```py + >>> tokenizer = AutoTokenizer.from_pretrained("./your/path/bigscience_t0") + >>> model = AutoModel.from_pretrained("./your/path/bigscience_t0") + ``` + +* Descarga de manera programática los archivos con la biblioteca [huggingface_hub](https://github.com/huggingface/huggingface_hub/tree/main/src/huggingface_hub): + + 1. Instala la biblioteca [huggingface_hub](https://github.com/huggingface/huggingface_hub/tree/main/src/huggingface_hub) en tu entorno virtual: + + ```bash + python -m pip install huggingface_hub + ``` + + 2. Utiliza la función [`hf_hub_download`](https://huggingface.co/docs/hub/adding-a-library#download-files-from-the-hub) para descargar un archivo a un path específico. Por ejemplo, el siguiente comando descarga el archivo `config.json` del modelo [T0](https://huggingface.co/bigscience/T0_3B) al path deseado: + + ```py + >>> from huggingface_hub import hf_hub_download + + >>> hf_hub_download(repo_id="bigscience/T0_3B", filename="config.json", cache_dir="./your/path/bigscience_t0") + ``` + +Una vez que el archivo se descargue y se almacene en caché localmente, especifica tu ruta local para cargarlo y usarlo: + +```py +>>> from transformers import AutoConfig + +>>> config = AutoConfig.from_pretrained("./your/path/bigscience_t0/config.json") +``` + + + +Para más detalles sobre cómo descargar archivos almacenados en el Hub consulta la sección [How to download files from the Hub](https://huggingface.co/docs/hub/how-to-downstream). + + + + + + + + From 6a4dbba1a3500507133825e2c5d194ab06c36e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Santiago=20G=C3=B3mez?= Date: Thu, 31 Mar 2022 06:45:18 -0500 Subject: [PATCH 35/54] Translate accelerate.mdx from english to spanish (#16176) * Translate accelerate.mdx from english to spanish * Update docs/source_es/accelerate.mdx Co-authored-by: Omar U. Espejel * Apply suggestions from code review Co-authored-by: Omar U. Espejel * Apply suggestions from code review Co-authored-by: Omar U. Espejel * Fix nits and finish translation Co-authored-by: Omar U. Espejel --- docs/source_es/accelerate.mdx | 132 ++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 docs/source_es/accelerate.mdx diff --git a/docs/source_es/accelerate.mdx b/docs/source_es/accelerate.mdx new file mode 100644 index 000000000000..59825cd81718 --- /dev/null +++ b/docs/source_es/accelerate.mdx @@ -0,0 +1,132 @@ + + +# Entrenamiento distribuido con 🤗 Accelerate + +El paralelismo ha emergido como una estrategia para entrenar modelos grandes en hardware limitado e incrementar la velocidad de entrenamiento en varios órdenes de magnitud. En Hugging Face creamos la biblioteca [🤗 Accelerate](https://huggingface.co/docs/accelerate/index.html) para ayudar a los usuarios a entrenar modelos 🤗 Transformers en cualquier tipo de configuración distribuida, ya sea en una máquina con múltiples GPUs o en múltiples GPUs distribuidas entre muchas máquinas. En este tutorial aprenderás cómo personalizar tu bucle de entrenamiento de PyTorch nativo para poder entrenar en entornos distribuidos. + +## Configuración + +Empecemos por instalar 🤗 Accelerate: + +```bash +pip install accelerate +``` + +Luego, importamos y creamos un objeto [`Accelerator`](https://huggingface.co/docs/accelerate/accelerator.html#accelerate.Accelerator). `Accelerator` detectará automáticamente el tipo de configuración distribuida que tengas disponible e inicializará todos los componentes necesarios para el entrenamiento. No necesitas especificar el dispositivo en donde se debe colocar tu modelo. + +```py +>>> from accelerate import Accelerator + +>>> accelerator = Accelerator() +``` + +## Prepárate para acelerar + +Pasa todos los objetos relevantes para el entrenamiento al método [`prepare`](https://huggingface.co/docs/accelerate/accelerator.html#accelerate.Accelerator.prepare). Esto incluye los DataLoaders de entrenamiento y evaluación, un modelo y un optimizador: + +```py +>>> train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare( +... train_dataloader, eval_dataloader, model, optimizer +... ) +``` + +## Backward + +Por último, reemplaza el típico `loss.backward()` en tu bucle de entrenamiento con el método [`backward`](https://huggingface.co/docs/accelerate/accelerator.html#accelerate.Accelerator.backward) de 🤗 Accelerate: + +```py +>>> for epoch in range(num_epochs): +... for batch in train_dataloader: +... outputs = model(**batch) +... loss = outputs.loss +... accelerator.backward(loss) + +... optimizer.step() +... lr_scheduler.step() +... optimizer.zero_grad() +... progress_bar.update(1) +``` + +Como se puede ver en el siguiente código, ¡solo necesitas adicionar cuatro líneas de código a tu bucle de entrenamiento para habilitar el entrenamiento distribuido! + +```diff ++ from accelerate import Accelerator + from transformers import AdamW, AutoModelForSequenceClassification, get_scheduler + ++ accelerator = Accelerator() + + model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2) + optimizer = AdamW(model.parameters(), lr=3e-5) + +- device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") +- model.to(device) + ++ train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare( ++ train_dataloader, eval_dataloader, model, optimizer ++ ) + + num_epochs = 3 + num_training_steps = num_epochs * len(train_dataloader) + lr_scheduler = get_scheduler( + "linear", + optimizer=optimizer, + num_warmup_steps=0, + num_training_steps=num_training_steps + ) + + progress_bar = tqdm(range(num_training_steps)) + + model.train() + for epoch in range(num_epochs): + for batch in train_dataloader: +- batch = {k: v.to(device) for k, v in batch.items()} + outputs = model(**batch) + loss = outputs.loss +- loss.backward() ++ accelerator.backward(loss) + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + progress_bar.update(1) +``` + +## Entrenamiento + +Una vez que hayas añadido las líneas de código relevantes, inicia el entrenamiento desde un script o notebook como Colaboratory. + +### Entrenar con un script + +Si estas corriendo tu entrenamiento desde un script ejecuta el siguiente comando para crear y guardar un archivo de configuración: + +```bash +accelerate config +``` + +Comienza el entrenamiento con: + +```bash +accelerate launch train.py +``` + +### Entrenar con un notebook + +🤗 Accelerate puede correr en un notebook si, por ejemplo, estás planeando utilizar las TPUs de Colaboratory. Encierra el código responsable del entrenamiento en una función y pásalo a `notebook_launcher`: + +```py +>>> from accelerate import notebook_launcher + +>>> notebook_launcher(training_function) +``` + +Para obtener más información sobre 🤗 Accelerate y sus numerosas funciones, consulta la [documentación](https://huggingface.co/docs/accelerate/index.html). From 05b4c32908fbc5914f930f912a5053e4f5f9e60c Mon Sep 17 00:00:00 2001 From: Bhadresh Savani Date: Thu, 31 Mar 2022 17:19:02 +0530 Subject: [PATCH 36/54] fixed a typo (#16508) --- examples/pytorch/question-answering/run_qa_no_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py index 2129ed97678f..964bcf63d175 100755 --- a/examples/pytorch/question-answering/run_qa_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_no_trainer.py @@ -791,7 +791,7 @@ def create_and_fill_np_array(start_or_end_logits, dataset, max_len): if not args.pad_to_max_length: # necessary to pad predictions and labels for being gathered start_logits = accelerator.pad_across_processes(start_logits, dim=1, pad_index=-100) - end_logits = accelerator.pad_across_processes(start_logits, dim=1, pad_index=-100) + end_logits = accelerator.pad_across_processes(end_logits, dim=1, pad_index=-100) all_start_logits.append(accelerator.gather(start_logits).cpu().numpy()) all_end_logits.append(accelerator.gather(end_logits).cpu().numpy()) From b808d8a5969ab7129a8374b732153ba7be2fbe2d Mon Sep 17 00:00:00 2001 From: Mowaninuola Osifeso <36759376+mowafess@users.noreply.github.com> Date: Thu, 31 Mar 2022 13:43:04 +0100 Subject: [PATCH 37/54] added type hints to xglm pytorch (#16500) * added type hints to xglm pytorch * Update src/transformers/models/xglm/modeling_xglm.py * Update src/transformers/models/xglm/modeling_xglm.py Co-authored-by: Matt --- src/transformers/models/xglm/modeling_xglm.py | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/transformers/models/xglm/modeling_xglm.py b/src/transformers/models/xglm/modeling_xglm.py index f07a68f06ed2..af277fcd7880 100755 --- a/src/transformers/models/xglm/modeling_xglm.py +++ b/src/transformers/models/xglm/modeling_xglm.py @@ -17,7 +17,7 @@ import math import random -from typing import Optional, Tuple +from typing import List, Optional, Tuple, Union import torch import torch.utils.checkpoint @@ -594,19 +594,19 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em ) def forward( self, - input_ids=None, - attention_mask=None, - encoder_hidden_states=None, - encoder_attention_mask=None, - head_mask=None, - cross_attn_head_mask=None, - past_key_values=None, - inputs_embeds=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): @@ -852,20 +852,20 @@ def set_output_embeddings(self, new_embeddings): ) def forward( self, - input_ids=None, - attention_mask=None, - encoder_hidden_states=None, - encoder_attention_mask=None, - head_mask=None, - cross_attn_head_mask=None, - past_key_values=None, - inputs_embeds=None, - labels=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., From e4b234834a79541f31be227aadce13f5aafda85a Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Thu, 31 Mar 2022 08:45:47 -0400 Subject: [PATCH 38/54] Fix syntax error in generate docstrings (#16516) --- src/transformers/generation_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/generation_utils.py b/src/transformers/generation_utils.py index 072ddfc3742a..1472052d2ba0 100644 --- a/src/transformers/generation_utils.py +++ b/src/transformers/generation_utils.py @@ -976,7 +976,7 @@ def generate( This value is subtracted from a beam's score if it generates a token same as any beam from other group at a particular time. Note that `diversity_penalty` is only effective if `group beam search` is enabled. - prefix_allowed_tokens_fn: (`Callable[[int, torch.Tensor], List[int]]`, *optional*): + prefix_allowed_tokens_fn (`Callable[[int, torch.Tensor], List[int]]`, *optional*): If provided, this function constraints the beam search to allowed tokens only at each step. If not provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned From 5807054bd346367efb8d771605f6aa0a4e46ca5b Mon Sep 17 00:00:00 2001 From: Anton Lozhkov Date: Thu, 31 Mar 2022 22:26:50 +0300 Subject: [PATCH 39/54] [research] link to the XTREME-S paper (#16519) * [research] link to the XTREME-S paper * Update examples/research_projects/xtreme-s/README.md Co-authored-by: Lysandre Debut Co-authored-by: Lysandre Debut --- examples/research_projects/xtreme-s/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/research_projects/xtreme-s/README.md b/examples/research_projects/xtreme-s/README.md index cfec5ff46328..dc7e783c75d1 100644 --- a/examples/research_projects/xtreme-s/README.md +++ b/examples/research_projects/xtreme-s/README.md @@ -20,9 +20,9 @@ limitations under the License. The Cross-lingual TRansfer Evaluation of Multilingual Encoders for Speech (XTREME-S) benchmark is a benchmark designed to evaluate speech representations across languages, tasks, domains and data regimes. It covers XX typologically diverse languages and seven downstream tasks grouped in four families: speech recognition, translation, classification and retrieval. -XTREME-S covers speech recognition with Fleurs, Multilingual LibriSpeech (MLS) and VoxPopuli, speech translation with CoVoST-2, speech classification with LangID (Fleurs) and intent classification (MInds-14) and finally speech(-text) retrieval with Fleurs. Each of the tasks covers a subset of the 102 languages included in XTREME-S (shown here with their ISO 3166-1 codes): afr, amh, ara, asm, ast, azj, bel, ben, bos, cat, ceb, zho_simpl, zho_trad, ces, cym, dan, deu, ell, eng, spa, est, fas, ful, fin, tgl, fra, gle, glg, guj, hau, heb, hin, hrv, hun, hye, ind, ibo, isl, ita, jpn, jav, kat, kam, kea, kaz, khm, kan, kor, ckb, kir, ltz, lug, lin, lao, lit, luo, lav, mri, mkd, mal, mon, mar, msa, mlt, mya, nob, npi, nld, nso, nya, oci, orm, ory, pan, pol, pus, por, ron, rus, bul, snd, slk, slv, sna, som, srp, swe, swh, tam, tel, tgk, tha, tur, ukr, umb, urd, uzb, vie, wol, xho, yor and zul. +XTREME-S covers speech recognition with Fleurs, Multilingual LibriSpeech (MLS) and VoxPopuli, speech translation with CoVoST-2, speech classification with LangID (Fleurs) and intent classification (MInds-14) and finally speech(-text) retrieval with Fleurs. Each of the tasks covers a subset of the 102 languages included in XTREME-S (shown here with their ISO 3166-1 codes): afr, amh, ara, asm, ast, azj, bel, ben, bos, cat, ceb, ces, cmn, cym, dan, deu, ell, eng, spa, est, fas, ful, fin, tgl, fra, gle, glg, guj, hau, heb, hin, hrv, hun, hye, ind, ibo, isl, ita, jpn, jav, kat, kam, kea, kaz, khm, kan, kor, ckb, kir, ltz, lug, lin, lao, lit, luo, lav, mri, mkd, mal, mon, mar, msa, mlt, mya, nob, npi, nld, nso, nya, oci, orm, ory, pan, pol, pus, por, ron, rus, bul, snd, slk, slv, sna, som, srp, swe, swh, tam, tel, tgk, tha, tur, ukr, umb, urd, uzb, vie, wol, xho, yor, yue and zul. -Paper: `` +Paper: [XTREME-S: Evaluating Cross-lingual Speech Representations](https://arxiv.org/abs/2203.10752) Dataset: [https://huggingface.co/datasets/google/xtreme_s](https://huggingface.co/datasets/google/xtreme_s) From bfeff6cc6a086b953b5f542d369b119e96b841cf Mon Sep 17 00:00:00 2001 From: Cathy <815244047@qq.com> Date: Fri, 1 Apr 2022 15:17:31 +0800 Subject: [PATCH 40/54] Fixed a typo in legacy seq2seq_trainer.py (#16531) --- examples/legacy/seq2seq/seq2seq_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/legacy/seq2seq/seq2seq_trainer.py b/examples/legacy/seq2seq/seq2seq_trainer.py index 1c2d7924a444..eeff082499c4 100644 --- a/examples/legacy/seq2seq/seq2seq_trainer.py +++ b/examples/legacy/seq2seq/seq2seq_trainer.py @@ -115,7 +115,7 @@ def create_optimizer_and_scheduler(self, num_training_steps: int): "eps": self.args.adam_epsilon, } optimizer_kwargs["lr"] = self.args.learning_rate - if self.sharded_dpp: + if self.sharded_ddp: self.optimizer = OSS( params=optimizer_grouped_parameters, optim=optimizer_cls, From 9de70f213eb234522095cc9af7b2fac53afc2d87 Mon Sep 17 00:00:00 2001 From: Jim Rohrer Date: Fri, 1 Apr 2022 03:52:42 -0500 Subject: [PATCH 41/54] Add ONNX export for BeiT (#16498) * Add beit onnx conversion support * Updated docs * Added cross reference to ViT ONNX config --- docs/source/serialization.mdx | 1 + src/transformers/models/beit/__init__.py | 4 ++-- .../models/beit/configuration_beit.py | 23 +++++++++++++++++++ src/transformers/onnx/features.py | 2 ++ tests/onnx/test_onnx_v2.py | 6 ++--- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/source/serialization.mdx b/docs/source/serialization.mdx index fc969aac4fdd..65fb5fa5cc54 100644 --- a/docs/source/serialization.mdx +++ b/docs/source/serialization.mdx @@ -47,6 +47,7 @@ Ready-made configurations include the following architectures: - ALBERT - BART +- BEiT - BERT - Blenderbot - BlenderbotSmall diff --git a/src/transformers/models/beit/__init__.py b/src/transformers/models/beit/__init__.py index 319fb2880a1d..27c31775d34e 100644 --- a/src/transformers/models/beit/__init__.py +++ b/src/transformers/models/beit/__init__.py @@ -22,7 +22,7 @@ _import_structure = { - "configuration_beit": ["BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "BeitConfig"], + "configuration_beit": ["BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "BeitConfig", "BeitOnnxConfig"], } if is_vision_available(): @@ -48,7 +48,7 @@ ] if TYPE_CHECKING: - from .configuration_beit import BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, BeitConfig + from .configuration_beit import BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, BeitConfig, BeitOnnxConfig if is_vision_available(): from .feature_extraction_beit import BeitFeatureExtractor diff --git a/src/transformers/models/beit/configuration_beit.py b/src/transformers/models/beit/configuration_beit.py index 9a1dfa8c20fc..7c47aba0c2ab 100644 --- a/src/transformers/models/beit/configuration_beit.py +++ b/src/transformers/models/beit/configuration_beit.py @@ -13,8 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. """ BEiT model configuration""" +from collections import OrderedDict +from typing import Mapping + +from packaging import version from ...configuration_utils import PretrainedConfig +from ...onnx import OnnxConfig from ...utils import logging @@ -176,3 +181,21 @@ def __init__( self.auxiliary_num_convs = auxiliary_num_convs self.auxiliary_concat_input = auxiliary_concat_input self.semantic_loss_ignore_index = semantic_loss_ignore_index + + +# Copied from transformers.models.vit.configuration_vit.ViTOnnxConfig +class BeitOnnxConfig(OnnxConfig): + + torch_onnx_minimum_version = version.parse("1.11") + + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + return OrderedDict( + [ + ("pixel_values", {0: "batch", 1: "sequence"}), + ] + ) + + @property + def atol_for_validation(self) -> float: + return 1e-4 diff --git a/src/transformers/onnx/features.py b/src/transformers/onnx/features.py index 926137c59482..cf5e55c521de 100644 --- a/src/transformers/onnx/features.py +++ b/src/transformers/onnx/features.py @@ -4,6 +4,7 @@ from .. import PretrainedConfig, PreTrainedModel, TFPreTrainedModel, is_tf_available, is_torch_available from ..models.albert import AlbertOnnxConfig from ..models.bart import BartOnnxConfig +from ..models.beit import BeitOnnxConfig from ..models.bert import BertOnnxConfig from ..models.blenderbot import BlenderbotOnnxConfig from ..models.blenderbot_small import BlenderbotSmallOnnxConfig @@ -270,6 +271,7 @@ class FeaturesManager: onnx_config_cls=ElectraOnnxConfig, ), "vit": supported_features_mapping("default", "image-classification", onnx_config_cls=ViTOnnxConfig), + "beit": supported_features_mapping("default", "image-classification", onnx_config_cls=BeitOnnxConfig), "blenderbot": supported_features_mapping( "default", "default-with-past", diff --git a/tests/onnx/test_onnx_v2.py b/tests/onnx/test_onnx_v2.py index f530515aed79..ba8d51158ff9 100644 --- a/tests/onnx/test_onnx_v2.py +++ b/tests/onnx/test_onnx_v2.py @@ -15,14 +15,13 @@ export, validate_model_outputs, ) +from transformers.onnx.utils import compute_effective_axis_dimension, compute_serialized_parameters_size +from transformers.testing_utils import require_onnx, require_tf, require_torch, require_vision, slow if is_torch_available() or is_tf_available(): from transformers.onnx.features import FeaturesManager -from transformers.onnx.utils import compute_effective_axis_dimension, compute_serialized_parameters_size -from transformers.testing_utils import require_onnx, require_tf, require_torch, require_vision, slow - @require_onnx class OnnxUtilsTestCaseV2(TestCase): @@ -181,6 +180,7 @@ def test_values_override(self): ("xlm-roberta", "xlm-roberta-base"), ("layoutlm", "microsoft/layoutlm-base-uncased"), ("vit", "google/vit-base-patch16-224"), + ("beit", "microsoft/beit-base-patch16-224"), } PYTORCH_EXPORT_WITH_PAST_MODELS = { From 483a9450a040a910ad26fd8cdcbc2f8e773cb161 Mon Sep 17 00:00:00 2001 From: Ferdinand Schlatt Date: Fri, 1 Apr 2022 14:50:47 +0200 Subject: [PATCH 42/54] call on_train_end when trial is pruned (#16536) --- src/transformers/trainer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 157e65d18352..948697e35127 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -991,6 +991,7 @@ def _report_to_hp_search( trial.report(self.objective, epoch) if trial.should_prune(): + self.callback_handler.on_train_end(self.args, self.state, self.control) raise optuna.TrialPruned() elif self.hp_search_backend == HPSearchBackend.RAY: from ray import tune From afc5a1ea3a91c486e458977d4bdbb85903909267 Mon Sep 17 00:00:00 2001 From: Dahlbomii <101373053+Dahlbomii@users.noreply.github.com> Date: Fri, 1 Apr 2022 06:27:41 -0700 Subject: [PATCH 43/54] Type hints added (#16529) --- .../models/openai/modeling_tf_openai.py | 98 ++++++++++--------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/src/transformers/models/openai/modeling_tf_openai.py b/src/transformers/models/openai/modeling_tf_openai.py index 490b3fac47e5..80d7a9abd192 100644 --- a/src/transformers/models/openai/modeling_tf_openai.py +++ b/src/transformers/models/openai/modeling_tf_openai.py @@ -16,8 +16,9 @@ """ TF 2.0 OpenAI GPT model.""" from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional, Tuple, Union +import numpy as np import tensorflow as tf from ...activations_tf import get_tf_activation @@ -25,6 +26,7 @@ from ...modeling_tf_utils import ( TFCausalLanguageModelingLoss, TFConv1D, + TFModelInputType, TFPreTrainedModel, TFSequenceClassificationLoss, TFSequenceSummary, @@ -510,18 +512,18 @@ def __init__(self, config, *inputs, **kwargs): ) def call( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, **kwargs, - ): + ) -> Union[Tuple, TFBaseModelOutput]: outputs = self.transformer( input_ids=input_ids, @@ -573,19 +575,19 @@ def set_output_embeddings(self, value): ) def call( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[Union[np.ndarray, tf.Tensor]] = None, + training: Optional[bool] = False, **kwargs, - ): + ) -> Union[Tuple, TFCausalLMOutput]: r""" labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., @@ -656,19 +658,19 @@ def __init__(self, config, *inputs, **kwargs): @replace_return_docstrings(output_type=TFOpenAIGPTDoubleHeadsModelOutput, config_class=_CONFIG_FOR_DOC) def call( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - mc_token_ids=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + mc_token_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, **kwargs, - ): + ) -> Union[Tuple, TFOpenAIGPTDoubleHeadsModelOutput]: r""" mc_token_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input): Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) - @@ -800,19 +802,19 @@ def __init__(self, config, *inputs, **kwargs): ) def call( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - training=False, + input_ids: Optional[TFModelInputType] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + token_type_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + position_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[Union[np.ndarray, tf.Tensor]] = None, + training: Optional[bool] = False, **kwargs, - ): + ) -> Union[Tuple, TFSequenceClassifierOutput]: r""" labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., From 59a9c83e40f879f5060eff99968dc688a56d0d0d Mon Sep 17 00:00:00 2001 From: Gunjan Chhablani Date: Fri, 1 Apr 2022 19:20:22 +0530 Subject: [PATCH 44/54] Fix Bart type hints (#16297) * Add type hints to PLBart PyTorch * Remove pending merge conflicts * Fix PLBart Type Hints * Add changes from review --- .../models/plbart/modeling_plbart.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/transformers/models/plbart/modeling_plbart.py b/src/transformers/models/plbart/modeling_plbart.py index b1a2088913fd..37230541e9db 100755 --- a/src/transformers/models/plbart/modeling_plbart.py +++ b/src/transformers/models/plbart/modeling_plbart.py @@ -16,7 +16,7 @@ import copy import math import random -from typing import List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import torch import torch.utils.checkpoint @@ -1142,21 +1142,21 @@ def get_decoder(self): ) def forward( self, - input_ids=None, - attention_mask=None, - decoder_input_ids=None, - decoder_attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, - encoder_outputs=None, - past_key_values=None, - inputs_embeds=None, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.LongTensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[List[torch.FloatTensor]] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, decoder_inputs_embeds=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, ): output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -1271,23 +1271,23 @@ def set_output_embeddings(self, new_embeddings): @add_end_docstrings(PLBART_GENERATION_EXAMPLE) def forward( self, - input_ids=None, - attention_mask=None, - decoder_input_ids=None, - decoder_attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, - encoder_outputs=None, - past_key_values=None, - inputs_embeds=None, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.LongTensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[List[torch.FloatTensor]] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, decoder_inputs_embeds=None, - labels=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], Seq2SeqLMOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., @@ -1345,16 +1345,16 @@ def forward( def prepare_inputs_for_generation( self, - decoder_input_ids, - past=None, - attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, - use_cache=None, - encoder_outputs=None, + decoder_input_ids: torch.LongTensor, + past: Optional[List[torch.FloatTensor]] = None, + attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + encoder_outputs: Optional[List[torch.FloatTensor]] = None, **kwargs # TODO: Check if this is needed. It is unused? - ): + ) -> Dict[str, Any]: # cut decoder_input_ids if past is used if past is not None: decoder_input_ids = decoder_input_ids[:, -1:] From 9947dd077c1dd3a4e220b1846ed38f475641e21d Mon Sep 17 00:00:00 2001 From: Gunjan Chhablani Date: Fri, 1 Apr 2022 19:32:58 +0530 Subject: [PATCH 45/54] Add VisualBert type hints (#16544) --- .../visual_bert/modeling_visual_bert.py | 184 +++++++++--------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/src/transformers/models/visual_bert/modeling_visual_bert.py b/src/transformers/models/visual_bert/modeling_visual_bert.py index 0e5acf32b3c4..69495785fe81 100755 --- a/src/transformers/models/visual_bert/modeling_visual_bert.py +++ b/src/transformers/models/visual_bert/modeling_visual_bert.py @@ -17,7 +17,7 @@ import math from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional, Tuple, Union import torch import torch.utils.checkpoint @@ -720,20 +720,20 @@ class PreTrainedModel @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - visual_embeds=None, - visual_attention_mask=None, - visual_token_type_ids=None, - image_text_alignment=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + visual_embeds: Optional[torch.FloatTensor] = None, + visual_attention_mask: Optional[torch.LongTensor] = None, + visual_token_type_ids: Optional[torch.LongTensor] = None, + image_text_alignment: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPooling]: r""" Returns: @@ -893,22 +893,22 @@ def set_output_embeddings(self, new_embeddings): @replace_return_docstrings(output_type=VisualBertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - visual_embeds=None, - visual_attention_mask=None, - visual_token_type_ids=None, - image_text_alignment=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - sentence_image_labels=None, - ): + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + visual_embeds: Optional[torch.FloatTensor] = None, + visual_attention_mask: Optional[torch.LongTensor] = None, + visual_token_type_ids: Optional[torch.LongTensor] = None, + image_text_alignment: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + sentence_image_labels: Optional[torch.LongTensor] = None, + ) -> Union[Tuple[torch.Tensor], VisualBertForPreTrainingOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., @@ -1039,21 +1039,21 @@ def __init__(self, config): @replace_return_docstrings(output_type=MultipleChoiceModelOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - visual_embeds=None, - visual_attention_mask=None, - visual_token_type_ids=None, - image_text_alignment=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + visual_embeds: Optional[torch.FloatTensor] = None, + visual_attention_mask: Optional[torch.LongTensor] = None, + visual_token_type_ids: Optional[torch.LongTensor] = None, + image_text_alignment: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., @@ -1191,21 +1191,21 @@ def __init__(self, config): @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - visual_embeds=None, - visual_attention_mask=None, - visual_token_type_ids=None, - image_text_alignment=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + visual_embeds: Optional[torch.FloatTensor] = None, + visual_attention_mask: Optional[torch.LongTensor] = None, + visual_token_type_ids: Optional[torch.LongTensor] = None, + image_text_alignment: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1317,21 +1317,21 @@ def __init__(self, config): @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - visual_embeds=None, - visual_attention_mask=None, - visual_token_type_ids=None, - image_text_alignment=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - labels=None, - ): + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + visual_embeds: Optional[torch.FloatTensor] = None, + visual_attention_mask: Optional[torch.LongTensor] = None, + visual_token_type_ids: Optional[torch.LongTensor] = None, + image_text_alignment: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., @@ -1477,22 +1477,22 @@ def __init__(self, config): @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) def forward( self, - input_ids=None, - attention_mask=None, - token_type_ids=None, - position_ids=None, - head_mask=None, - inputs_embeds=None, - visual_embeds=None, - visual_attention_mask=None, - visual_token_type_ids=None, - image_text_alignment=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - region_to_phrase_position=None, - labels=None, - ): + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + visual_embeds: Optional[torch.FloatTensor] = None, + visual_attention_mask: Optional[torch.LongTensor] = None, + visual_token_type_ids: Optional[torch.LongTensor] = None, + image_text_alignment: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + region_to_phrase_position: Optional[torch.LongTensor] = None, + labels: Optional[torch.LongTensor] = None, + ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: r""" region_to_phrase_position (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*): The positions depicting the position of the image embedding corresponding to the textual tokens. From 5fe06b9bdda55cb3ed891c6989414a1c5f26431b Mon Sep 17 00:00:00 2001 From: Rishav Chandra Varma Date: Fri, 1 Apr 2022 19:51:26 +0530 Subject: [PATCH 46/54] Adding missing type hints for mBART model (PyTorch) (#16429) * added type hints for mbart tensorflow tf implementation * Adding missing type hints for mBART model Tensorflow Implementation model added with missing type hints * Missing Type hints - correction For TF model * Code fixup using make quality tests * Hint types - typo error * make fix-copies and make fixup * type hints * updated files * type hints update * making dependent modesls coherent Co-authored-by: matt --- .../modeling_bigbird_pegasus.py | 2 +- .../models/blenderbot/modeling_blenderbot.py | 4 +- .../models/m2m_100/modeling_m2m_100.py | 4 +- .../models/mbart/modeling_mbart.py | 112 +++++++++--------- .../models/pegasus/modeling_pegasus.py | 4 +- src/transformers/models/xglm/modeling_xglm.py | 2 +- 6 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py index 540f77944b7b..1fb8de8e1452 100755 --- a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py +++ b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py @@ -1478,7 +1478,7 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = True, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* diff --git a/src/transformers/models/blenderbot/modeling_blenderbot.py b/src/transformers/models/blenderbot/modeling_blenderbot.py index 928e22e860e7..d1f84d2c3917 100755 --- a/src/transformers/models/blenderbot/modeling_blenderbot.py +++ b/src/transformers/models/blenderbot/modeling_blenderbot.py @@ -294,7 +294,7 @@ def forward( attention_mask: torch.Tensor, layer_head_mask: torch.Tensor, output_attentions: bool = False, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* @@ -378,7 +378,7 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = True, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* diff --git a/src/transformers/models/m2m_100/modeling_m2m_100.py b/src/transformers/models/m2m_100/modeling_m2m_100.py index 3bb749564a01..d816218824e1 100755 --- a/src/transformers/models/m2m_100/modeling_m2m_100.py +++ b/src/transformers/models/m2m_100/modeling_m2m_100.py @@ -363,7 +363,7 @@ def forward( attention_mask: torch.Tensor, layer_head_mask: torch.Tensor, output_attentions: bool = False, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* @@ -447,7 +447,7 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = True, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* diff --git a/src/transformers/models/mbart/modeling_mbart.py b/src/transformers/models/mbart/modeling_mbart.py index 446a02f648cd..6ed7c24ab176 100755 --- a/src/transformers/models/mbart/modeling_mbart.py +++ b/src/transformers/models/mbart/modeling_mbart.py @@ -307,7 +307,7 @@ def forward( attention_mask: torch.Tensor, layer_head_mask: torch.Tensor, output_attentions: bool = False, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* @@ -390,7 +390,7 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = True, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* @@ -722,14 +722,14 @@ def _backward_compatibility_gradient_checkpointing(self): def forward( self, - input_ids=None, - attention_mask=None, - head_mask=None, - inputs_embeds=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): @@ -913,19 +913,19 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em def forward( self, - input_ids=None, - attention_mask=None, - encoder_hidden_states=None, - encoder_attention_mask=None, - head_mask=None, - cross_attn_head_mask=None, - past_key_values=None, - inputs_embeds=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): @@ -1168,22 +1168,22 @@ def get_decoder(self): ) def forward( self, - input_ids=None, - attention_mask=None, - decoder_input_ids=None, - decoder_attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, - encoder_outputs=None, - past_key_values=None, - inputs_embeds=None, - decoder_inputs_embeds=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Seq2SeqModelOutput, Tuple[torch.FloatTensor]]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1297,23 +1297,23 @@ def set_output_embeddings(self, new_embeddings): @add_end_docstrings(MBART_GENERATION_EXAMPLE) def forward( self, - input_ids=None, - attention_mask=None, - decoder_input_ids=None, - decoder_attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, - encoder_outputs=None, - past_key_values=None, - inputs_embeds=None, - decoder_inputs_embeds=None, - labels=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - ): + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Seq2SeqLMOutput, Tuple[torch.FloatTensor]]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., diff --git a/src/transformers/models/pegasus/modeling_pegasus.py b/src/transformers/models/pegasus/modeling_pegasus.py index f1d7a6ce56ef..06cf9f130a73 100755 --- a/src/transformers/models/pegasus/modeling_pegasus.py +++ b/src/transformers/models/pegasus/modeling_pegasus.py @@ -309,7 +309,7 @@ def forward( attention_mask: torch.Tensor, layer_head_mask: torch.Tensor, output_attentions: bool = False, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* @@ -393,7 +393,7 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = True, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* diff --git a/src/transformers/models/xglm/modeling_xglm.py b/src/transformers/models/xglm/modeling_xglm.py index af277fcd7880..8d45e2b200b7 100755 --- a/src/transformers/models/xglm/modeling_xglm.py +++ b/src/transformers/models/xglm/modeling_xglm.py @@ -423,7 +423,7 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = True, - ): + ) -> torch.Tensor: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)* From 823dbf8a41a243c8a1c7ee9f231f1142e312c53d Mon Sep 17 00:00:00 2001 From: Gunjan Chhablani Date: Fri, 1 Apr 2022 20:09:28 +0530 Subject: [PATCH 47/54] Remove MBart subclass of XLMRoberta in tokenzier docs (#16546) * Remove MBart subclass of XLMRoberta in tokenzier * Fix style * Copy docs from MBart50 tokenizer --- src/transformers/models/mbart/tokenization_mbart_fast.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/mbart/tokenization_mbart_fast.py b/src/transformers/models/mbart/tokenization_mbart_fast.py index 1de8d62f3608..a172d37913a4 100644 --- a/src/transformers/models/mbart/tokenization_mbart_fast.py +++ b/src/transformers/models/mbart/tokenization_mbart_fast.py @@ -62,9 +62,8 @@ class MBartTokenizerFast(PreTrainedTokenizerFast): Construct a "fast" MBART tokenizer (backed by HuggingFace's *tokenizers* library). Based on [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models). - [`MBartTokenizerFast`] is a subclass of [`XLMRobertaTokenizerFast`]. Refer to superclass - [`XLMRobertaTokenizerFast`] for usage examples and documentation concerning the initialization parameters and other - methods. + This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should + refer to this superclass for more information regarding those methods. The tokenization method is ` ` for source language documents, and `` ``` for target language documents. From 2199382dfd6c1e010518acdd5732adc114a6a7f0 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 1 Apr 2022 16:53:07 +0200 Subject: [PATCH 48/54] Use random_attention_mask for TF tests (#16517) * use random_attention_mask for TF tests * Fix for TFCLIP test (for now). Co-authored-by: ydshieh --- ...est_modeling_tf_{{cookiecutter.lowercase_modelname}}.py | 4 ++-- tests/albert/test_modeling_tf_albert.py | 4 ++-- tests/bert/test_modeling_tf_bert.py | 4 ++-- tests/clip/test_modeling_tf_clip.py | 6 ++++++ tests/convbert/test_modeling_tf_convbert.py | 4 ++-- tests/ctrl/test_modeling_tf_ctrl.py | 4 ++-- tests/deberta/test_modeling_tf_deberta.py | 4 ++-- tests/deberta_v2/test_modeling_tf_deberta_v2.py | 4 ++-- tests/distilbert/test_modeling_tf_distilbert.py | 4 ++-- tests/dpr/test_modeling_tf_dpr.py | 7 +++---- tests/electra/test_modeling_tf_electra.py | 4 ++-- tests/flaubert/test_modeling_tf_flaubert.py | 4 ++-- tests/funnel/test_modeling_tf_funnel.py | 4 ++-- tests/gpt2/test_modeling_tf_gpt2.py | 4 ++-- tests/gptj/test_modeling_tf_gptj.py | 4 ++-- tests/layoutlm/test_modeling_tf_layoutlm.py | 4 ++-- tests/longformer/test_modeling_tf_longformer.py | 4 ++-- tests/lxmert/test_modeling_tf_lxmert.py | 4 ++-- tests/mobilebert/test_modeling_tf_mobilebert.py | 4 ++-- tests/mpnet/test_modeling_tf_mpnet.py | 4 ++-- tests/openai/test_modeling_tf_openai.py | 4 ++-- tests/rembert/test_modeling_tf_rembert.py | 4 ++-- tests/roberta/test_modeling_tf_roberta.py | 4 ++-- tests/roformer/test_modeling_tf_roformer.py | 4 ++-- tests/t5/test_modeling_tf_t5.py | 4 ++-- tests/tapas/test_modeling_tf_tapas.py | 4 ++-- tests/test_modeling_tf_common.py | 2 +- tests/xlm/test_modeling_tf_xlm.py | 4 ++-- tests/xlnet/test_modeling_tf_xlnet.py | 4 ++-- 29 files changed, 62 insertions(+), 57 deletions(-) diff --git a/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/test_modeling_tf_{{cookiecutter.lowercase_modelname}}.py b/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/test_modeling_tf_{{cookiecutter.lowercase_modelname}}.py index 16b31500dd6c..57fd95dd3ff6 100644 --- a/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/test_modeling_tf_{{cookiecutter.lowercase_modelname}}.py +++ b/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/test_modeling_tf_{{cookiecutter.lowercase_modelname}}.py @@ -21,7 +21,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask if is_tf_available(): @@ -92,7 +92,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/albert/test_modeling_tf_albert.py b/tests/albert/test_modeling_tf_albert.py index 59815561c056..7eacc1f32a47 100644 --- a/tests/albert/test_modeling_tf_albert.py +++ b/tests/albert/test_modeling_tf_albert.py @@ -21,7 +21,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -96,7 +96,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/bert/test_modeling_tf_bert.py b/tests/bert/test_modeling_tf_bert.py index 611268337ffd..8c709e093801 100644 --- a/tests/bert/test_modeling_tf_bert.py +++ b/tests/bert/test_modeling_tf_bert.py @@ -21,7 +21,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask from ..utils.test_modeling_tf_core import TFCoreModelTesterMixin @@ -96,7 +96,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/clip/test_modeling_tf_clip.py b/tests/clip/test_modeling_tf_clip.py index 02e289cd5b2a..d3c3cb9f5033 100644 --- a/tests/clip/test_modeling_tf_clip.py +++ b/tests/clip/test_modeling_tf_clip.py @@ -301,6 +301,12 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: input_mask = random_attention_mask([self.batch_size, self.seq_length]) + # make sure the first token has attention mask `1` to ensure that, after combining the causal mask, there + # is still at least one token being attended to for each batch. + # TODO: Change `random_attention_mask` in PT/TF/Flax common test file, after a discussion with the team. + input_mask = tf.concat( + [tf.ones_like(input_mask[:, :1], dtype=input_mask.dtype), input_mask[:, 1:]], axis=-1 + ) config = self.get_config() diff --git a/tests/convbert/test_modeling_tf_convbert.py b/tests/convbert/test_modeling_tf_convbert.py index ff4cbb1aa974..e2d68876263a 100644 --- a/tests/convbert/test_modeling_tf_convbert.py +++ b/tests/convbert/test_modeling_tf_convbert.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -94,7 +94,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/ctrl/test_modeling_tf_ctrl.py b/tests/ctrl/test_modeling_tf_ctrl.py index 65b984b51c9a..d17a97a3ad83 100644 --- a/tests/ctrl/test_modeling_tf_ctrl.py +++ b/tests/ctrl/test_modeling_tf_ctrl.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -69,7 +69,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/deberta/test_modeling_tf_deberta.py b/tests/deberta/test_modeling_tf_deberta.py index 581f6f02f470..7e2a3c3110ee 100644 --- a/tests/deberta/test_modeling_tf_deberta.py +++ b/tests/deberta/test_modeling_tf_deberta.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -92,7 +92,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/deberta_v2/test_modeling_tf_deberta_v2.py b/tests/deberta_v2/test_modeling_tf_deberta_v2.py index 391afee59784..4fd967c2fa6e 100644 --- a/tests/deberta_v2/test_modeling_tf_deberta_v2.py +++ b/tests/deberta_v2/test_modeling_tf_deberta_v2.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -95,7 +95,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/distilbert/test_modeling_tf_distilbert.py b/tests/distilbert/test_modeling_tf_distilbert.py index 7a146e9c3bf8..5266723f1f86 100644 --- a/tests/distilbert/test_modeling_tf_distilbert.py +++ b/tests/distilbert/test_modeling_tf_distilbert.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -70,7 +70,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) sequence_labels = None token_labels = None diff --git a/tests/dpr/test_modeling_tf_dpr.py b/tests/dpr/test_modeling_tf_dpr.py index 7a48a2254e10..ffce36efc3a6 100644 --- a/tests/dpr/test_modeling_tf_dpr.py +++ b/tests/dpr/test_modeling_tf_dpr.py @@ -19,7 +19,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -94,9 +94,8 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor( - [self.batch_size, self.seq_length], vocab_size=2 - ) # follow test_modeling_tf_ctrl.py + # follow test_modeling_tf_ctrl.py + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/electra/test_modeling_tf_electra.py b/tests/electra/test_modeling_tf_electra.py index 4593ecff6100..ff2acd37e69f 100644 --- a/tests/electra/test_modeling_tf_electra.py +++ b/tests/electra/test_modeling_tf_electra.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask if is_tf_available(): @@ -71,7 +71,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/flaubert/test_modeling_tf_flaubert.py b/tests/flaubert/test_modeling_tf_flaubert.py index 62503bac2861..86bcd6ea6484 100644 --- a/tests/flaubert/test_modeling_tf_flaubert.py +++ b/tests/flaubert/test_modeling_tf_flaubert.py @@ -19,7 +19,7 @@ from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -75,7 +75,7 @@ def __init__( def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) - input_mask = ids_tensor([self.batch_size, self.seq_length], 2, dtype=tf.float32) + input_mask = random_attention_mask([self.batch_size, self.seq_length], dtype=tf.float32) input_lengths = None if self.use_input_lengths: diff --git a/tests/funnel/test_modeling_tf_funnel.py b/tests/funnel/test_modeling_tf_funnel.py index 6105f9ab8035..c3ae3788d61e 100644 --- a/tests/funnel/test_modeling_tf_funnel.py +++ b/tests/funnel/test_modeling_tf_funnel.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -111,7 +111,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/gpt2/test_modeling_tf_gpt2.py b/tests/gpt2/test_modeling_tf_gpt2.py index f94387509e6a..d6470c0d1526 100644 --- a/tests/gpt2/test_modeling_tf_gpt2.py +++ b/tests/gpt2/test_modeling_tf_gpt2.py @@ -19,7 +19,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask from ..utils.test_modeling_tf_core import TFCoreModelTesterMixin @@ -74,7 +74,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/gptj/test_modeling_tf_gptj.py b/tests/gptj/test_modeling_tf_gptj.py index 32ce3f8564b0..63feffb8c62e 100644 --- a/tests/gptj/test_modeling_tf_gptj.py +++ b/tests/gptj/test_modeling_tf_gptj.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow, tooslow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask from ..utils.test_modeling_tf_core import TFCoreModelTesterMixin @@ -70,7 +70,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/layoutlm/test_modeling_tf_layoutlm.py b/tests/layoutlm/test_modeling_tf_layoutlm.py index f60d0c6f91d5..90e2b4fcf169 100644 --- a/tests/layoutlm/test_modeling_tf_layoutlm.py +++ b/tests/layoutlm/test_modeling_tf_layoutlm.py @@ -21,7 +21,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -107,7 +107,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/longformer/test_modeling_tf_longformer.py b/tests/longformer/test_modeling_tf_longformer.py index 37c1ce534953..6bfa708912dd 100644 --- a/tests/longformer/test_modeling_tf_longformer.py +++ b/tests/longformer/test_modeling_tf_longformer.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -79,7 +79,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/lxmert/test_modeling_tf_lxmert.py b/tests/lxmert/test_modeling_tf_lxmert.py index 8d91d249d90b..63ec44a1ad90 100644 --- a/tests/lxmert/test_modeling_tf_lxmert.py +++ b/tests/lxmert/test_modeling_tf_lxmert.py @@ -23,7 +23,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -124,7 +124,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_lang_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) diff --git a/tests/mobilebert/test_modeling_tf_mobilebert.py b/tests/mobilebert/test_modeling_tf_mobilebert.py index 4cbfcefee874..c0ddf043562f 100644 --- a/tests/mobilebert/test_modeling_tf_mobilebert.py +++ b/tests/mobilebert/test_modeling_tf_mobilebert.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -114,7 +114,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/mpnet/test_modeling_tf_mpnet.py b/tests/mpnet/test_modeling_tf_mpnet.py index 23448610cc21..f9f9e2d51201 100644 --- a/tests/mpnet/test_modeling_tf_mpnet.py +++ b/tests/mpnet/test_modeling_tf_mpnet.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -90,7 +90,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) sequence_labels = None token_labels = None diff --git a/tests/openai/test_modeling_tf_openai.py b/tests/openai/test_modeling_tf_openai.py index 227689df59aa..f74a85ee60d6 100644 --- a/tests/openai/test_modeling_tf_openai.py +++ b/tests/openai/test_modeling_tf_openai.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -70,7 +70,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/rembert/test_modeling_tf_rembert.py b/tests/rembert/test_modeling_tf_rembert.py index f8f17f30a9dd..d5d52062e8c9 100644 --- a/tests/rembert/test_modeling_tf_rembert.py +++ b/tests/rembert/test_modeling_tf_rembert.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask if is_tf_available(): @@ -95,7 +95,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/roberta/test_modeling_tf_roberta.py b/tests/roberta/test_modeling_tf_roberta.py index fa947d64f081..9771673d8748 100644 --- a/tests/roberta/test_modeling_tf_roberta.py +++ b/tests/roberta/test_modeling_tf_roberta.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask if is_tf_available(): @@ -72,7 +72,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/roformer/test_modeling_tf_roformer.py b/tests/roformer/test_modeling_tf_roformer.py index 1f26f7e2adc6..9a23ca3b83d2 100644 --- a/tests/roformer/test_modeling_tf_roformer.py +++ b/tests/roformer/test_modeling_tf_roformer.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -95,7 +95,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = None if self.use_token_type_ids: diff --git a/tests/t5/test_modeling_tf_t5.py b/tests/t5/test_modeling_tf_t5.py index a2ea255faca5..c6585f83b18e 100644 --- a/tests/t5/test_modeling_tf_t5.py +++ b/tests/t5/test_modeling_tf_t5.py @@ -20,7 +20,7 @@ from transformers.utils import cached_property from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -58,7 +58,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_labels = None if self.use_labels: diff --git a/tests/tapas/test_modeling_tf_tapas.py b/tests/tapas/test_modeling_tf_tapas.py index 936273a6ca30..9e3cb63f70b5 100644 --- a/tests/tapas/test_modeling_tf_tapas.py +++ b/tests/tapas/test_modeling_tf_tapas.py @@ -38,7 +38,7 @@ from transformers.utils import cached_property from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -158,7 +158,7 @@ def prepare_config_and_inputs(self): input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + input_mask = random_attention_mask([self.batch_size, self.seq_length]) token_type_ids = [] for type_vocab_size in self.type_vocab_sizes: diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 3d2f7976cf6c..9473a50f53aa 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -1440,7 +1440,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None): def random_attention_mask(shape, rng=None, name=None, dtype=None): attn_mask = ids_tensor(shape, vocab_size=2, rng=None, name=None, dtype=dtype) # make sure that at least one token is attended to for each batch - attn_mask = tf.concat([tf.constant(value=1, shape=(shape[0], 1), dtype=dtype), attn_mask[:, 1:]], axis=1) + attn_mask = tf.concat([attn_mask[:, :-1], tf.ones_like(attn_mask[:, -1:], dtype=dtype)], axis=-1) return attn_mask diff --git a/tests/xlm/test_modeling_tf_xlm.py b/tests/xlm/test_modeling_tf_xlm.py index 5fc4d2413f9e..412a8430ad6d 100644 --- a/tests/xlm/test_modeling_tf_xlm.py +++ b/tests/xlm/test_modeling_tf_xlm.py @@ -20,7 +20,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -75,7 +75,7 @@ def __init__( def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) - input_mask = ids_tensor([self.batch_size, self.seq_length], 2, dtype=tf.float32) + input_mask = random_attention_mask([self.batch_size, self.seq_length], dtype=tf.float32) input_lengths = None if self.use_input_lengths: diff --git a/tests/xlnet/test_modeling_tf_xlnet.py b/tests/xlnet/test_modeling_tf_xlnet.py index 4b92581a0efc..8cf4ca2099bd 100644 --- a/tests/xlnet/test_modeling_tf_xlnet.py +++ b/tests/xlnet/test_modeling_tf_xlnet.py @@ -22,7 +22,7 @@ from transformers.testing_utils import require_tf, slow from ..test_configuration_common import ConfigTester -from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask if is_tf_available(): @@ -75,7 +75,7 @@ def prepare_config_and_inputs(self): input_ids_1 = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) input_ids_2 = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) segment_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) - input_mask = ids_tensor([self.batch_size, self.seq_length], 2, dtype=tf.float32) + input_mask = random_attention_mask([self.batch_size, self.seq_length], dtype=tf.float32) input_ids_q = ids_tensor([self.batch_size, self.seq_length + 1], self.vocab_size) perm_mask = tf.zeros((self.batch_size, self.seq_length + 1, self.seq_length), dtype=tf.float32) From 61ee26a89260611c51a277b69155710316ae54ac Mon Sep 17 00:00:00 2001 From: NielsRogge <48327001+NielsRogge@users.noreply.github.com> Date: Fri, 1 Apr 2022 17:19:36 +0200 Subject: [PATCH 49/54] Improve code example (#16450) Co-authored-by: Niels Rogge --- src/transformers/models/glpn/modeling_glpn.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/glpn/modeling_glpn.py b/src/transformers/models/glpn/modeling_glpn.py index c8d6bac79b36..86e53c787572 100755 --- a/src/transformers/models/glpn/modeling_glpn.py +++ b/src/transformers/models/glpn/modeling_glpn.py @@ -708,18 +708,36 @@ def forward( ```python >>> from transformers import GLPNFeatureExtractor, GLPNForDepthEstimation + >>> import torch + >>> import numpy as np >>> from PIL import Image >>> import requests - >>> feature_extractor = GLPNFeatureExtractor.from_pretrained("vinvino02/glpn-kitti") - >>> model = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-kitti") - >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) + >>> feature_extractor = GLPNFeatureExtractor.from_pretrained("vinvino02/glpn-kitti") + >>> model = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-kitti") + + >>> # prepare image for the model >>> inputs = feature_extractor(images=image, return_tensors="pt") - >>> outputs = model(**inputs) - >>> predicted_depth = outputs.predicted_depth # shape (batch_size, height, width) + + >>> with torch.no_grad(): + ... outputs = model(**inputs) + ... predicted_depth = outputs.predicted_depth + + >>> # interpolate to original size + >>> prediction = torch.nn.functional.interpolate( + ... predicted_depth.unsqueeze(1), + ... size=image.size[::-1], + ... mode="bicubic", + ... align_corners=False, + ... ) + + >>> # visualize the prediction + >>> output = prediction.squeeze().cpu().numpy() + >>> formatted = (output * 255 / np.max(output)).astype("uint8") + >>> depth = Image.fromarray(formatted) ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict output_hidden_states = ( From 53a4d6b115bea63f740c352a97ef5f3b88a279e2 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Fri, 1 Apr 2022 17:53:18 +0200 Subject: [PATCH 50/54] Pin tokenizers version <0.13 (#16539) * Pin tokenizers version <0.13 * Style --- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c9455eaa901d..56ba7d4c4784 100644 --- a/setup.py +++ b/setup.py @@ -151,7 +151,7 @@ "tf2onnx", "timeout-decorator", "timm", - "tokenizers>=0.11.1,!=0.11.3", + "tokenizers>=0.11.1,!=0.11.3,<0.13", "torch>=1.0", "torchaudio", "pyctcdecode>=0.3.0", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 2ba72f5b9593..334103c20a56 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -61,7 +61,7 @@ "tf2onnx": "tf2onnx", "timeout-decorator": "timeout-decorator", "timm": "timm", - "tokenizers": "tokenizers>=0.11.1,!=0.11.3", + "tokenizers": "tokenizers>=0.11.1,!=0.11.3,<0.13", "torch": "torch>=1.0", "torchaudio": "torchaudio", "pyctcdecode": "pyctcdecode>=0.3.0", From 60d27b1f152c181705191765661967fef3016cef Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 1 Apr 2022 17:54:01 +0200 Subject: [PATCH 51/54] Add code samples for TF speech models (#16494) Co-authored-by: ydshieh --- src/transformers/utils/doc.py | 63 +++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/transformers/utils/doc.py b/src/transformers/utils/doc.py index f395f8d4fb80..eaf59ba50215 100644 --- a/src/transformers/utils/doc.py +++ b/src/transformers/utils/doc.py @@ -794,6 +794,67 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): ``` """ +TF_SPEECH_BASE_MODEL_SAMPLE = r""" + Example: + + ```python + >>> from transformers import {processor_class}, {model_class} + >>> from datasets import load_dataset + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation") + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> processor = {processor_class}.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="tf") + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + {expected_output} + ``` +""" + +TF_SPEECH_CTC_SAMPLE = r""" + Example: + + ```python + >>> from transformers import {processor_class}, {model_class} + >>> from datasets import load_dataset + >>> import tensorflow as tf + + >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation") + >>> dataset = dataset.sort("id") + >>> sampling_rate = dataset.features["audio"].sampling_rate + + >>> processor = {processor_class}.from_pretrained("{checkpoint}") + >>> model = {model_class}.from_pretrained("{checkpoint}") + + >>> # audio file is decoded on the fly + >>> inputs = processor(dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="tf") + >>> logits = model(**inputs).logits + >>> predicted_ids = tf.math.argmax(logits, axis=-1) + + >>> # transcribe speech + >>> transcription = processor.batch_decode(predicted_ids) + >>> transcription[0] + {expected_output} + ``` + + ```python + >>> with processor.as_target_processor(): + ... inputs["labels"] = processor(dataset[0]["text"], return_tensors="tf").input_ids + + >>> # compute loss + >>> loss = model(**inputs).loss + >>> round(float(loss), 2) + {expected_loss} + ``` +""" + TF_VISION_BASE_MODEL_SAMPLE = r""" Example: @@ -848,6 +909,8 @@ def _prepare_output_docstrings(output_type, config_class, min_indent=None): "MaskedLM": TF_MASKED_LM_SAMPLE, "LMHead": TF_CAUSAL_LM_SAMPLE, "BaseModel": TF_BASE_MODEL_SAMPLE, + "SpeechBaseModel": TF_SPEECH_BASE_MODEL_SAMPLE, + "CTC": TF_SPEECH_CTC_SAMPLE, "VisionBaseModel": TF_VISION_BASE_MODEL_SAMPLE, "ImageClassification": TF_VISION_SEQ_CLASS_SAMPLE, } From 9ac49ab51f2aba80b528a7a5678a94d987572cee Mon Sep 17 00:00:00 2001 From: Johannes Kolbe Date: Mon, 28 Mar 2022 21:54:56 +0200 Subject: [PATCH 52/54] add doctests to TF ViT --- src/transformers/models/vit/modeling_tf_vit.py | 5 +++-- utils/documentation_tests.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py index e2e946d8c9f4..36d3bbc69a07 100644 --- a/src/transformers/models/vit/modeling_tf_vit.py +++ b/src/transformers/models/vit/modeling_tf_vit.py @@ -662,7 +662,6 @@ def call( Returns: Examples: - ```python >>> from transformers import ViTFeatureExtractor, TFViTModel >>> from PIL import Image @@ -677,6 +676,8 @@ def call( >>> inputs = feature_extractor(images=image, return_tensors="tf") >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + [1, 197, 768] ```""" outputs = self.vit( @@ -768,7 +769,6 @@ def call( Returns: Examples: - ```python >>> from transformers import ViTFeatureExtractor, TFViTForImageClassification >>> import tensorflow as tf @@ -787,6 +787,7 @@ def call( >>> # model predicts one of the 1000 ImageNet classes >>> predicted_class_idx = tf.math.argmax(logits, axis=-1)[0] >>> print("Predicted class:", model.config.id2label[int(predicted_class_idx)]) + Predicted class: Egyptian cat ```""" outputs = self.vit( diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 372e63ad232b..86ee4c02a560 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -46,6 +46,7 @@ src/transformers/models/van/modeling_van.py src/transformers/models/vilt/modeling_vilt.py src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py src/transformers/models/vit/modeling_vit.py +src/transformers/models/vit/modeling_tf_vit.py src/transformers/models/vit_mae/modeling_vit_mae.py src/transformers/models/wav2vec2/modeling_wav2vec2.py src/transformers/models/wav2vec2/tokenization_wav2vec2.py From f4f0f6c448e6f4a7fa47d514d36ccf5b2c32c352 Mon Sep 17 00:00:00 2001 From: Johannes Kolbe Date: Mon, 28 Mar 2022 22:34:13 +0200 Subject: [PATCH 53/54] add empty lines back in --- src/transformers/models/vit/modeling_tf_vit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py index 36d3bbc69a07..1831d77754eb 100644 --- a/src/transformers/models/vit/modeling_tf_vit.py +++ b/src/transformers/models/vit/modeling_tf_vit.py @@ -662,6 +662,7 @@ def call( Returns: Examples: + ```python >>> from transformers import ViTFeatureExtractor, TFViTModel >>> from PIL import Image @@ -769,6 +770,7 @@ def call( Returns: Examples: + ```python >>> from transformers import ViTFeatureExtractor, TFViTForImageClassification >>> import tensorflow as tf From 92e0b2dc7a4aed306c1e9c7159de4578dade5e68 Mon Sep 17 00:00:00 2001 From: Johannes Kolbe Date: Sat, 2 Apr 2022 00:09:54 +0200 Subject: [PATCH 54/54] use add_code_sample_docstrings decorator --- .../models/vit/modeling_tf_vit.py | 82 ++++++------------- 1 file changed, 26 insertions(+), 56 deletions(-) diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py index 1831d77754eb..9e17f19ba62e 100644 --- a/src/transformers/models/vit/modeling_tf_vit.py +++ b/src/transformers/models/vit/modeling_tf_vit.py @@ -33,14 +33,23 @@ unpack_inputs, ) from ...tf_utils import shape_list -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_vit import ViTConfig logger = logging.get_logger(__name__) +# General docstring _CONFIG_FOR_DOC = "ViTConfig" -_CHECKPOINT_FOR_DOC = "google/vit-base-patch16-224" +_FEAT_EXTRACTOR_FOR_DOC = "ViTFeatureExtractor" + +# Base docstring +_CHECKPOINT_FOR_DOC = "google/vit-base-patch16-224-in21k" +_EXPECTED_OUTPUT_SHAPE = [1, 197, 768] + +# Image classification docstring +_IMAGE_CLASS_CHECKPOINT = "google/vit-base-patch16-224" +_IMAGE_CLASS_EXPECTED_OUTPUT = "Egyptian cat" # Inspired by @@ -646,7 +655,14 @@ def __init__(self, config: ViTConfig, *inputs, add_pooling_layer=True, **kwargs) @unpack_inputs @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TFBaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + @add_code_sample_docstrings( + processor_class=_FEAT_EXTRACTOR_FOR_DOC, + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TFBaseModelOutputWithPooling, + config_class=_CONFIG_FOR_DOC, + modality="vision", + expected_output=_EXPECTED_OUTPUT_SHAPE, + ) def call( self, pixel_values: Optional[TFModelInputType] = None, @@ -658,28 +674,6 @@ def call( training: bool = False, **kwargs, ) -> Union[TFBaseModelOutputWithPooling, Tuple[tf.Tensor]]: - r""" - Returns: - - Examples: - - ```python - >>> from transformers import ViTFeatureExtractor, TFViTModel - >>> from PIL import Image - >>> import requests - - >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" - >>> image = Image.open(requests.get(url, stream=True).raw) - - >>> feature_extractor = ViTFeatureExtractor.from_pretrained("google/vit-base-patch16-224-in21k") - >>> model = TFViTModel.from_pretrained("google/vit-base-patch16-224-in21k") - - >>> inputs = feature_extractor(images=image, return_tensors="tf") - >>> outputs = model(**inputs) - >>> last_hidden_states = outputs.last_hidden_state - >>> list(last_hidden_states.shape) - [1, 197, 768] - ```""" outputs = self.vit( pixel_values=pixel_values, @@ -748,7 +742,13 @@ def __init__(self, config: ViTConfig, *inputs, **kwargs): @unpack_inputs @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TFSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + @add_code_sample_docstrings( + processor_class=_FEAT_EXTRACTOR_FOR_DOC, + checkpoint=_IMAGE_CLASS_CHECKPOINT, + output_type=TFSequenceClassifierOutput, + config_class=_CONFIG_FOR_DOC, + expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, + ) def call( self, pixel_values: Optional[TFModelInputType] = None, @@ -761,36 +761,6 @@ def call( training: Optional[bool] = False, **kwargs, ) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]: - r""" - labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*): - Labels for computing the image classification/regression loss. Indices should be in `[0, ..., - config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If - `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - - Returns: - - Examples: - - ```python - >>> from transformers import ViTFeatureExtractor, TFViTForImageClassification - >>> import tensorflow as tf - >>> from PIL import Image - >>> import requests - - >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" - >>> image = Image.open(requests.get(url, stream=True).raw) - - >>> feature_extractor = ViTFeatureExtractor.from_pretrained("google/vit-base-patch16-224") - >>> model = TFViTForImageClassification.from_pretrained("google/vit-base-patch16-224") - - >>> inputs = feature_extractor(images=image, return_tensors="tf") - >>> outputs = model(**inputs) - >>> logits = outputs.logits - >>> # model predicts one of the 1000 ImageNet classes - >>> predicted_class_idx = tf.math.argmax(logits, axis=-1)[0] - >>> print("Predicted class:", model.config.id2label[int(predicted_class_idx)]) - Predicted class: Egyptian cat - ```""" outputs = self.vit( pixel_values=pixel_values,