From 4ec1c82b4e781e57dfb189172997b341d6c69041 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 2 Mar 2023 13:21:56 +0530 Subject: [PATCH 001/277] Add FlaxWhisperForAudioClassification model --- docs/source/en/model_doc/whisper.mdx | 6 ++ .../models/auto/modeling_flax_auto.py | 6 ++ src/transformers/models/whisper/__init__.py | 2 + .../models/whisper/modeling_flax_whisper.py | 92 +++++++++++++++++++ .../whisper/test_modeling_flax_whisper.py | 7 +- 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/docs/source/en/model_doc/whisper.mdx b/docs/source/en/model_doc/whisper.mdx index 22b08e4e61bc..e2176fdde9e4 100644 --- a/docs/source/en/model_doc/whisper.mdx +++ b/docs/source/en/model_doc/whisper.mdx @@ -105,3 +105,9 @@ The original code can be found [here](https://github.com/openai/whisper). [[autodoc]] FlaxWhisperForConditionalGeneration - __call__ + +## WhisperForAudioClassification + +[[autodoc]] WhisperForAudioClassification + - forward + diff --git a/src/transformers/models/auto/modeling_flax_auto.py b/src/transformers/models/auto/modeling_flax_auto.py index 77be9b33f0a7..b9c413ee1636 100644 --- a/src/transformers/models/auto/modeling_flax_auto.py +++ b/src/transformers/models/auto/modeling_flax_auto.py @@ -225,6 +225,12 @@ ] ) +FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES = OrderedDict( + [ + ("whisper", "FlaxWhisperForAudioClassification"), + ] +) + FLAX_MODEL_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_MAPPING_NAMES) FLAX_MODEL_FOR_PRETRAINING_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_PRETRAINING_MAPPING_NAMES) diff --git a/src/transformers/models/whisper/__init__.py b/src/transformers/models/whisper/__init__.py index 3b6015a56f6f..cd962478e34d 100644 --- a/src/transformers/models/whisper/__init__.py +++ b/src/transformers/models/whisper/__init__.py @@ -75,6 +75,7 @@ "FlaxWhisperForConditionalGeneration", "FlaxWhisperModel", "FlaxWhisperPreTrainedModel", + "FlaxWhisperForAudioClassification", ] @@ -126,6 +127,7 @@ pass else: from .modeling_flax_whisper import ( + FlaxWhisperForAudioClassification, FlaxWhisperForConditionalGeneration, FlaxWhisperModel, FlaxWhisperPreTrainedModel, diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index a928e145e8aa..45ecbbe52aa9 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -35,6 +35,7 @@ FlaxCausalLMOutputWithCrossAttentions, FlaxSeq2SeqLMOutput, FlaxSeq2SeqModelOutput, + FlaxSequenceClassifierOutput, ) from ...modeling_flax_utils import ( ACT2FN, @@ -1468,3 +1469,94 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): append_replace_return_docstrings( FlaxWhisperForConditionalGeneration, output_type=FlaxSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC ) + + +class FlaxWhisperForAudioClassificationModule(nn.Module): + config: WhisperConfig + dtype: jnp.dtype = jnp.float32 + gradient_checkpointing: bool = False + + def setup(self) -> None: + self.encoder = FlaxWhisperEncoder(config=self.config, dtype=self.dtype) + num_layers = self.config.num_hidden_layers + 1 + if self.config.use_weighted_layer_sum: + self.layer_weights = jnp.repeat(1 / num_layers, num_layers) + self.projector = nn.Dense(self.config.classifier_proj_size) + self.classifier = nn.Dense(self.config.num_labels, dtype=self.dtype) + + def __call__( + self, + input_features, + encoder_outputs, + labels, + output_attenions, + output_hidden_states: bool = True, + return_dict: bool = True, + ): + output_attentions = output_attenions if output_attenions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_features, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if self.config.use_weighted_layer_sum: + hidden_states = jnp.stack(encoder_outputs, dim=1) + norm_weights = jax.nn.softmax(self.layer_weights, axis=-1) + hidden_states = jnp.sum(hidden_states * jnp.reshape(norm_weights, [-1, 1, 1]), axis=1) + else: + hidden_states = encoder_outputs[0] + + hidden_states = self.projector(hidden_states) + pooled_output = jnp.mean(hidden_states, axis=1) + + logits = self.classifier(pooled_output) + + if not return_dict: + return (logits,) + encoder_outputs[1:] + + return FlaxSequenceClassifierOutput( + logits=logits, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +@add_start_docstrings("The Whisper Model with an audio classification head on top.", WHISPER_START_DOCSTRING) +class FlaxWhisperForAudioClassification(FlaxWhisperPreTrainedModel): + module_class = FlaxWhisperForAudioClassificationModule + dtype: jnp.dtype = jnp.float32 + + +FLAX_WHISPER_AUDIO_CLASSIFICATION_DOCSTRING = r""" + Returns: + + Transcription example: + + ```python + >>> from transformers import WhisperProcessor, FlaxWhisperForAudioClassification + >>> from datasets import load_dataset + + >>> processor = WhisperProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = FlaxWhisperForAudioClassification.from_pretrained("openai/whisper-tiny.en", from_pt=True) + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="np") + >>> input_features = inputs.input_features + >>> outputs = model(input_features=input_features) + >>> logits = outputs.logit + ``` +""" + +overwrite_call_docstring( + FlaxWhisperForAudioClassification, WHISPER_INPUTS_DOCSTRING + FLAX_WHISPER_AUDIO_CLASSIFICATION_DOCSTRING +) +append_replace_return_docstrings( + FlaxWhisperForAudioClassification, output_type=FlaxSequenceClassifierOutput, config_class=_CONFIG_FOR_DOC +) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index 3f1e201d72d8..f2ccba7680f1 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -21,6 +21,7 @@ import transformers from transformers import WhisperConfig, is_flax_available +from transformers.models.whisper import FlaxWhisperForAudioClassification from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow from transformers.utils import cached_property from transformers.utils.import_utils import is_datasets_available @@ -192,7 +193,11 @@ def make_partial_class(full_class, *args, **kwargs): @require_flax class FlaxWhisperModelTest(FlaxModelTesterMixin, unittest.TestCase): - all_model_classes = (FlaxWhisperForConditionalGeneration, FlaxWhisperModel) if is_flax_available() else () + all_model_classes = ( + (FlaxWhisperForConditionalGeneration, FlaxWhisperModel, FlaxWhisperForAudioClassification) + if is_flax_available() + else () + ) all_generative_model_classes = (FlaxWhisperForConditionalGeneration,) if is_flax_available() else () is_encoder_decoder = True test_pruning = False From bab776724de1a7fe81fbd70523a8c2c5fa950101 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 2 Mar 2023 13:41:39 +0530 Subject: [PATCH 002/277] Fix check_repo failures --- docs/source/en/model_doc/whisper.mdx | 6 +++--- src/transformers/__init__.py | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/source/en/model_doc/whisper.mdx b/docs/source/en/model_doc/whisper.mdx index e2176fdde9e4..52a8b5953c63 100644 --- a/docs/source/en/model_doc/whisper.mdx +++ b/docs/source/en/model_doc/whisper.mdx @@ -106,8 +106,8 @@ The original code can be found [here](https://github.com/openai/whisper). [[autodoc]] FlaxWhisperForConditionalGeneration - __call__ -## WhisperForAudioClassification +## FlaxWhisperForAudioClassification -[[autodoc]] WhisperForAudioClassification - - forward +[[autodoc]] FlaxWhisperForAudioClassification + - __call__ diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index fafe8ba287d4..e9725b7b5608 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -3627,6 +3627,7 @@ "FlaxWhisperForConditionalGeneration", "FlaxWhisperModel", "FlaxWhisperPreTrainedModel", + "FlaxWhisperForAudioClassification", ] ) _import_structure["models.xglm"].extend( @@ -6620,7 +6621,12 @@ FlaxWav2Vec2Model, FlaxWav2Vec2PreTrainedModel, ) - from .models.whisper import FlaxWhisperForConditionalGeneration, FlaxWhisperModel, FlaxWhisperPreTrainedModel + from .models.whisper import ( + FlaxWhisperForConditionalGeneration, + FlaxWhisperModel, + FlaxWhisperPreTrainedModel, + FlaxWhisperForAudioClassification + ) from .models.xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel from .models.xlm_roberta import ( FLAX_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST, From 963820878688bf83ef12b6dad4533425f57f5743 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 2 Mar 2023 13:49:42 +0530 Subject: [PATCH 003/277] fix-copies --- src/transformers/utils/dummy_flax_objects.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/transformers/utils/dummy_flax_objects.py b/src/transformers/utils/dummy_flax_objects.py index 60004790ec35..b5746f5f9604 100644 --- a/src/transformers/utils/dummy_flax_objects.py +++ b/src/transformers/utils/dummy_flax_objects.py @@ -1161,6 +1161,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) +class FlaxWhisperForAudioClassificatio(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + class FlaxXGLMForCausalLM(metaclass=DummyObject): _backends = ["flax"] From 92fd2d83dad683ff42638433c0eed5e5e5eab582 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 2 Mar 2023 14:04:36 +0530 Subject: [PATCH 004/277] fix-copies --- src/transformers/utils/dummy_flax_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/utils/dummy_flax_objects.py b/src/transformers/utils/dummy_flax_objects.py index b5746f5f9604..4eebf3653bd3 100644 --- a/src/transformers/utils/dummy_flax_objects.py +++ b/src/transformers/utils/dummy_flax_objects.py @@ -1161,7 +1161,7 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) -class FlaxWhisperForAudioClassificatio(metaclass=DummyObject): +class FlaxWhisperForAudioClassification(metaclass=DummyObject): _backends = ["flax"] def __init__(self, *args, **kwargs): From baf7554729cad3f05b7c099f2f440ba806610eac Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 2 Mar 2023 14:17:37 +0530 Subject: [PATCH 005/277] Fix copies --- src/transformers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index e9725b7b5608..706a243027f3 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -6625,7 +6625,7 @@ FlaxWhisperForConditionalGeneration, FlaxWhisperModel, FlaxWhisperPreTrainedModel, - FlaxWhisperForAudioClassification + FlaxWhisperForAudioClassification, ) from .models.xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel from .models.xlm_roberta import ( From 16f1cb9ae335294fbd6bd247719f9ac34c0dba0c Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 2 Mar 2023 14:25:13 +0530 Subject: [PATCH 006/277] Fix ruff failure --- src/transformers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 706a243027f3..6b16212ee790 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -6622,10 +6622,10 @@ FlaxWav2Vec2PreTrainedModel, ) from .models.whisper import ( + FlaxWhisperForAudioClassification, FlaxWhisperForConditionalGeneration, FlaxWhisperModel, FlaxWhisperPreTrainedModel, - FlaxWhisperForAudioClassification, ) from .models.xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel from .models.xlm_roberta import ( From 3b4fddb634d73a6734d2f3e5593eef6f1264dbb9 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 2 Mar 2023 14:30:22 +0530 Subject: [PATCH 007/277] Fix copies --- src/transformers/utils/dummy_flax_objects.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/utils/dummy_flax_objects.py b/src/transformers/utils/dummy_flax_objects.py index 4eebf3653bd3..31ce1e22d62c 100644 --- a/src/transformers/utils/dummy_flax_objects.py +++ b/src/transformers/utils/dummy_flax_objects.py @@ -1140,28 +1140,28 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) -class FlaxWhisperForConditionalGeneration(metaclass=DummyObject): +class FlaxWhisperForAudioClassification(metaclass=DummyObject): _backends = ["flax"] def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) -class FlaxWhisperModel(metaclass=DummyObject): +class FlaxWhisperForConditionalGeneration(metaclass=DummyObject): _backends = ["flax"] def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) -class FlaxWhisperPreTrainedModel(metaclass=DummyObject): +class FlaxWhisperModel(metaclass=DummyObject): _backends = ["flax"] def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) -class FlaxWhisperForAudioClassification(metaclass=DummyObject): +class FlaxWhisperPreTrainedModel(metaclass=DummyObject): _backends = ["flax"] def __init__(self, *args, **kwargs): From 7cb70bdb1a38a48ff311e61ccde50d28b5ec159e Mon Sep 17 00:00:00 2001 From: raghavanone Date: Fri, 3 Mar 2023 17:51:33 +0530 Subject: [PATCH 008/277] WIP introduce FlaxWhisperEncoder tests --- .../models/whisper/modeling_flax_whisper.py | 35 +++- .../whisper/test_modeling_flax_whisper.py | 195 +++++++++++++++++- 2 files changed, 220 insertions(+), 10 deletions(-) diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index 45ecbbe52aa9..7f5fb76e5b11 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -1487,10 +1487,10 @@ def setup(self) -> None: def __call__( self, input_features, - encoder_outputs, - labels, - output_attenions, + encoder_outputs=None, + output_attenions=None, output_hidden_states: bool = True, + decoder_input_ids=None, return_dict: bool = True, ): output_attentions = output_attenions if output_attenions is not None else self.config.output_attentions @@ -1534,6 +1534,35 @@ class FlaxWhisperForAudioClassification(FlaxWhisperPreTrainedModel): module_class = FlaxWhisperForAudioClassificationModule dtype: jnp.dtype = jnp.float32 + def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: + # init input tensors + input_features = jnp.zeros(input_shape, dtype="f4") + input_features = input_features.at[(..., -1)].set(self.config.eos_token_id) + + decoder_input_ids = jnp.zeros((input_shape[0], 1), dtype="i4") + jnp.ones_like(decoder_input_ids) + + batch_size, sequence_length = decoder_input_ids.shape + jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) + + params_rng, dropout_rng = jax.random.split(rng) + rngs = {"params": params_rng, "dropout": dropout_rng} + + random_params = self.module.init( + rngs, + input_features=input_features, + )["params"] + + if params is not None: + random_params = flatten_dict(unfreeze(random_params)) + params = flatten_dict(unfreeze(params)) + for missing_key in self._missing_keys: + params[missing_key] = random_params[missing_key] + self._missing_keys = set() + return freeze(unflatten_dict(params)) + else: + return random_params + FLAX_WHISPER_AUDIO_CLASSIFICATION_DOCSTRING = r""" Returns: diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index f2ccba7680f1..42f3bad66f5d 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -12,8 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - +import copy import functools import inspect import tempfile @@ -193,11 +192,7 @@ def make_partial_class(full_class, *args, **kwargs): @require_flax class FlaxWhisperModelTest(FlaxModelTesterMixin, unittest.TestCase): - all_model_classes = ( - (FlaxWhisperForConditionalGeneration, FlaxWhisperModel, FlaxWhisperForAudioClassification) - if is_flax_available() - else () - ) + all_model_classes = (FlaxWhisperForConditionalGeneration, FlaxWhisperModel) if is_flax_available() else () all_generative_model_classes = (FlaxWhisperForConditionalGeneration,) if is_flax_available() else () is_encoder_decoder = True test_pruning = False @@ -709,3 +704,189 @@ def test_tiny_timestamp_generation(self): transcript = processor.batch_decode(generated_ids, skip_special_tokens=True, output_offsets=True) self.assertEqual(transcript, EXPECTED_TRANSCRIPT) + + +def prepare_whisper_encoder_inputs_dict( + config, + input_features, +): + return { + "input_features": input_features, + } + + +class FlaxWhisperEncoderModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=60, + is_training=True, + use_labels=True, + hidden_size=16, + num_hidden_layers=2, + num_attention_heads=4, + input_channels=1, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=20, + max_source_positions=1500, + num_mel_bins=80, + num_conv_layers=1, + suppress_tokens=None, + begin_suppress_tokens=None, + classifier_proj_size=4, + num_labels=2, + is_encoder_decoder=False, + is_decoder=False, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_labels = use_labels + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.input_channels = input_channels + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.num_mel_bins = num_mel_bins + self.max_position_embeddings = max_position_embeddings + self.max_source_positions = max_source_positions + self.num_conv_layers = num_conv_layers + self.suppress_tokens = suppress_tokens + self.begin_suppress_tokens = begin_suppress_tokens + self.classifier_proj_size = classifier_proj_size + self.num_labels = num_labels + self.is_encoder_decoder = is_encoder_decoder + self.is_decoder = is_decoder + + def get_config(self): + return WhisperConfig( + d_model=self.hidden_size, + encoder_layers=self.num_hidden_layers, + decoder_layers=self.num_hidden_layers, + encoder_attention_heads=self.num_attention_heads, + decoder_attention_heads=self.num_attention_heads, + input_channels=self.input_channels, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + max_source_positions=self.max_source_positions, + decoder_ffn_dim=self.hidden_size, + encoder_ffn_dim=self.hidden_size, + suppress_tokens=self.suppress_tokens, + begin_suppress_tokens=self.begin_suppress_tokens, + classifier_proj_size=self.classifier_proj_size, + num_labels=self.num_labels, + is_encoder_decoder=self.is_encoder_decoder, + is_decoder=self.is_decoder, + ) + + def prepare_config_and_inputs(self): + input_features = floats_tensor([self.batch_size, self.num_mel_bins, self.seq_length]) + + config = self.get_config() + inputs_dict = prepare_whisper_encoder_inputs_dict( + config, + input_features=input_features, + ) + return config, inputs_dict + + def prepare_config_and_inputs_for_common(self): + config, inputs_dict = self.prepare_config_and_inputs() + return config, inputs_dict + + def get_subsampled_output_lengths(self, input_lengths): + """ + Computes the output length of the convolutional layers + """ + + for i in range(self.num_conv_layers): + input_lengths = (input_lengths - 1) // 2 + 1 + + return input_lengths + + @property + def encoder_seq_length(self): + return self.get_subsampled_output_lengths(self.seq_length) + + def create_and_check_model_forward(self, config, inputs_dict, freeze_encoder=False): + model = FlaxWhisperForAudioClassification(config=config) + + if freeze_encoder: + model.freeze_encoder() + + input_features = inputs_dict["input_features"] + + # first forward pass + last_hidden_state = model(input_features).logits + + self.parent.assertTrue(last_hidden_state.shape, (13, 2)) + + +@require_flax +class WhisperEncoderModelTest(FlaxModelTesterMixin, unittest.TestCase): + all_model_classes = (FlaxWhisperForAudioClassification,) if is_flax_available() else () + is_encoder_decoder = False + fx_compatible = False + test_pruning = False + test_missing_keys = False + + input_name = "input_features" + + def setUp(self): + self.model_tester = FlaxWhisperEncoderModelTester(self) + self.config_tester = ConfigTester(self, config_class=WhisperConfig) + self.maxDiff = 3000 + + def test_config(self): + self.config_tester.run_common_tests() + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["input_features", "head_mask", "encoder_outputs"] + self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names) + + # input embeds is meaningless for an encoder-only acoustic model + def test_inputs_embeds(self): + pass + + # the equivalent test is passing the encoder outputs directly to the model + def test_encoder_outputs(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + + inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) + + outputs = model(**inputs)[0] + + input_ids = inputs["input_features"] + del inputs["input_features"] + + encoder = model.encoder + + inputs["encoder_outputs"] = encoder(input_ids) + outputs_embeds = model(**inputs)[0] + + self.assertTrue((outputs_embeds == outputs).all()) + + # WhisperEncoder has no inputs_embeds and thus the `get_input_embeddings` fn is not implemented + def test_model_common_attributes(self): + pass + + # WhisperEncoder cannot resize token embeddings since it has no tokens embeddings + def test_resize_tokens_embeddings(self): + pass From b1c71bed9a3b1d00e817f811606350d29718f43e Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 9 Mar 2023 17:19:28 +0530 Subject: [PATCH 009/277] Remove decoder Ids from the model input --- src/transformers/models/whisper/modeling_flax_whisper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index 7f5fb76e5b11..4dbef6418b52 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -1490,7 +1490,6 @@ def __call__( encoder_outputs=None, output_attenions=None, output_hidden_states: bool = True, - decoder_input_ids=None, return_dict: bool = True, ): output_attentions = output_attenions if output_attenions is not None else self.config.output_attentions From f7163f4b328e5d7f1da2a520f9b86646627204a5 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 9 Mar 2023 17:28:38 +0530 Subject: [PATCH 010/277] Fix importing of FlaxWhisperForAudioClassification --- tests/models/whisper/test_modeling_flax_whisper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index 42f3bad66f5d..abafa58e3b02 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -20,7 +20,6 @@ import transformers from transformers import WhisperConfig, is_flax_available -from transformers.models.whisper import FlaxWhisperForAudioClassification from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow from transformers.utils import cached_property from transformers.utils.import_utils import is_datasets_available @@ -42,6 +41,7 @@ from transformers import ( FLAX_MODEL_MAPPING, FlaxWhisperForConditionalGeneration, + FlaxWhisperForAudioClassification, FlaxWhisperModel, WhisperFeatureExtractor, WhisperProcessor, From d35b75247237a3966050ded59d4a66ae0cb325e1 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 9 Mar 2023 17:35:20 +0530 Subject: [PATCH 011/277] Fix style failure --- tests/models/whisper/test_modeling_flax_whisper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index abafa58e3b02..b8fc7fc6ede1 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -40,8 +40,8 @@ from transformers import ( FLAX_MODEL_MAPPING, - FlaxWhisperForConditionalGeneration, FlaxWhisperForAudioClassification, + FlaxWhisperForConditionalGeneration, FlaxWhisperModel, WhisperFeatureExtractor, WhisperProcessor, From 3bd9f82eb1f28d19c0f65490d6b3758ac361bd89 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 9 Mar 2023 18:09:47 +0530 Subject: [PATCH 012/277] Fix FlaxWhisperForAudioClassification init --- .../models/whisper/test_modeling_flax_whisper.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index b8fc7fc6ede1..b05309f35252 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -731,7 +731,7 @@ def __init__( hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, max_position_embeddings=20, - max_source_positions=1500, + max_source_positions=30, num_mel_bins=80, num_conv_layers=1, suppress_tokens=None, @@ -838,10 +838,20 @@ class WhisperEncoderModelTest(FlaxModelTesterMixin, unittest.TestCase): input_name = "input_features" + # def setUp(self): + # self.model_tester = FlaxWhisperEncoderModelTester(self) + # self.config_tester = ConfigTester(self, config_class=WhisperConfig) + # self.maxDiff = 3000 + def setUp(self): - self.model_tester = FlaxWhisperEncoderModelTester(self) + self.model_tester = FlaxWhisperModelTester(self) + _, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + self.init_shape = (1,) + inputs_dict["input_features"].shape[1:] + + self.all_model_classes = ( + make_partial_class(model_class, input_shape=self.init_shape) for model_class in self.all_model_classes + ) self.config_tester = ConfigTester(self, config_class=WhisperConfig) - self.maxDiff = 3000 def test_config(self): self.config_tester.run_common_tests() From 54458ef4c4e89d5ff708984dfaeb9bf4e1c6908c Mon Sep 17 00:00:00 2001 From: raghavanone Date: Fri, 10 Mar 2023 13:29:39 +0530 Subject: [PATCH 013/277] Fix tests --- .../models/whisper/modeling_flax_whisper.py | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index 4dbef6418b52..23f66e2e64d5 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -1478,6 +1478,7 @@ class FlaxWhisperForAudioClassificationModule(nn.Module): def setup(self) -> None: self.encoder = FlaxWhisperEncoder(config=self.config, dtype=self.dtype) + self.config.is_encoder_decoder = False num_layers = self.config.num_hidden_layers + 1 if self.config.use_weighted_layer_sum: self.layer_weights = jnp.repeat(1 / num_layers, num_layers) @@ -1488,11 +1489,11 @@ def __call__( self, input_features, encoder_outputs=None, - output_attenions=None, + output_attentions=None, output_hidden_states: bool = True, return_dict: bool = True, ): - output_attentions = output_attenions if output_attenions is not None else self.config.output_attentions + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) @@ -1562,6 +1563,60 @@ def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: Froz else: return random_params + def __call__( + self, + input_features: jnp.ndarray, + attention_mask: Optional[jnp.ndarray] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + train: bool = False, + params: dict = None, + dropout_rng: PRNGKey = None, + **kwargs, + ): + r""" + Returns: + + Example: + + ```python + >>> from transformers import WhisperProcessor, FlaxWhisperForConditionalGeneration + >>> from datasets import load_dataset + + >>> processor = WhisperProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = FlaxWhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en", from_pt=True) + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="np") + >>> input_features = inputs.input_features + >>> encoder_outputs = model.encode(input_features=input_features) + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + # Handle any PRNG if needed + rngs = {} + if dropout_rng is not None: + rngs["dropout"] = dropout_rng + + def _encoder_forward(module, input_features, **kwargs): + encode_module = module._get_encoder_module() + return encode_module(input_features, **kwargs) + + return self.module.apply( + {"params": params or self.params}, + input_features=jnp.array(input_features, dtype="f4"), + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + rngs=rngs, + # method=_encoder_forward, + ) + + FLAX_WHISPER_AUDIO_CLASSIFICATION_DOCSTRING = r""" Returns: From 3d037a0dc63a58f5454235a83b4fa517623ae813 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Fri, 10 Mar 2023 16:14:37 +0530 Subject: [PATCH 014/277] Fix tests --- .../models/auto/modeling_flax_auto.py | 7 ++ .../models/whisper/modeling_flax_whisper.py | 1 + .../whisper/test_modeling_flax_whisper.py | 87 +++++++++++++++---- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/src/transformers/models/auto/modeling_flax_auto.py b/src/transformers/models/auto/modeling_flax_auto.py index b9c413ee1636..80e1a1d09ab4 100644 --- a/src/transformers/models/auto/modeling_flax_auto.py +++ b/src/transformers/models/auto/modeling_flax_auto.py @@ -261,6 +261,9 @@ FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING = _LazyAutoMapping( CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES ) +FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING = _LazyAutoMapping( + CONFIG_MAPPING_NAMES, FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES +) class FlaxAutoModel(_BaseAutoModelClass): @@ -361,6 +364,10 @@ class FlaxAutoModelForSpeechSeq2Seq(_BaseAutoModelClass): _model_mapping = FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING +class FlaxAutoModelForAudioClassification(_BaseAutoModelClass): + _model_mapping = FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING + + FlaxAutoModelForSpeechSeq2Seq = auto_class_update( FlaxAutoModelForSpeechSeq2Seq, head_doc="sequence-to-sequence speech-to-text modeling" ) diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index 23f66e2e64d5..9c6f1a1de945 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -1563,6 +1563,7 @@ def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: Froz else: return random_params + @add_start_docstrings_to_model_forward(WHISPER_INPUTS_DOCSTRING) def __call__( self, input_features: jnp.ndarray, diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index b05309f35252..02679d13a7ae 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -856,42 +856,93 @@ def setUp(self): def test_config(self): self.config_tester.run_common_tests() + # overwrite because of `input_features` + def test_jit_compilation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + prepared_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + model = model_class(config) + + @jax.jit + def model_jitted(input_features, decoder_input_ids, **kwargs): + return model(input_features=input_features, decoder_input_ids=decoder_input_ids, **kwargs) + + with self.subTest("JIT Enabled"): + jitted_outputs = model_jitted(**prepared_inputs_dict).to_tuple() + + with self.subTest("JIT Disabled"): + with jax.disable_jit(): + outputs = model_jitted(**prepared_inputs_dict).to_tuple() + + self.assertEqual(len(outputs), len(jitted_outputs)) + for jitted_output, output in zip(jitted_outputs, outputs): + self.assertEqual(jitted_output.shape, output.shape) + + # overwrite because of `input_features` + def test_save_load_from_base(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + base_class = make_partial_class(FLAX_MODEL_MAPPING[config.__class__], input_shape=self.init_shape) + + for model_class in self.all_model_classes: + if model_class.__name__ == base_class.__name__: + continue + + model = base_class(config) + base_params = flatten_dict(unfreeze(model.params)) + + # check that all base model weights are loaded correctly + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + head_model = model_class.from_pretrained(tmpdirname) + + base_param_from_head = flatten_dict(unfreeze(head_model.params[head_model.base_model_prefix])) + # base_param_from_head = get_params(head_model.params, from_head_prefix=head_model.base_model_prefix) + + for key in base_param_from_head.keys(): + max_diff = (base_params[key] - base_param_from_head[key]).sum().item() + self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() for model_class in self.all_model_classes: model = model_class(config) - signature = inspect.signature(model.forward) + signature = inspect.signature(model.__call__) # signature.parameters is an OrderedDict => so arg_names order is deterministic arg_names = [*signature.parameters.keys()] - expected_arg_names = ["input_features", "head_mask", "encoder_outputs"] + expected_arg_names = ["input_features", "attention_mask", "output_attentions"] self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names) - # input embeds is meaningless for an encoder-only acoustic model - def test_inputs_embeds(self): - pass - - # the equivalent test is passing the encoder outputs directly to the model - def test_encoder_outputs(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + # overwrite because of `input_features` + def test_save_load_to_base(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + base_class = make_partial_class(FLAX_MODEL_MAPPING[config.__class__], input_shape=self.init_shape) for model_class in self.all_model_classes: - model = model_class(config) + if model_class.__name__ == base_class.__name__: + continue - inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) + model = model_class(config) + base_params_from_head = flatten_dict(unfreeze(model.params[model.base_model_prefix])) - outputs = model(**inputs)[0] + # check that all base model weights are loaded correctly + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + base_model = base_class.from_pretrained(tmpdirname) - input_ids = inputs["input_features"] - del inputs["input_features"] + base_params = flatten_dict(unfreeze(base_model.params)) - encoder = model.encoder + for key in base_params_from_head.keys(): + max_diff = (base_params[key] - base_params_from_head[key]).sum().item() + self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - inputs["encoder_outputs"] = encoder(input_ids) - outputs_embeds = model(**inputs)[0] - self.assertTrue((outputs_embeds == outputs).all()) + # input embeds is meaningless for an encoder-only acoustic model + def test_inputs_embeds(self): + pass # WhisperEncoder has no inputs_embeds and thus the `get_input_embeddings` fn is not implemented def test_model_common_attributes(self): From 8707310e77bdd8e079efe9773b000de9fa4a7db9 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Tue, 14 Mar 2023 12:45:34 +0530 Subject: [PATCH 015/277] Fix style related issues --- src/transformers/models/whisper/modeling_flax_whisper.py | 1 - tests/models/whisper/test_modeling_flax_whisper.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index 9c6f1a1de945..0b45f2b164f1 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -1618,7 +1618,6 @@ def _encoder_forward(module, input_features, **kwargs): ) - FLAX_WHISPER_AUDIO_CLASSIFICATION_DOCSTRING = r""" Returns: diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index 02679d13a7ae..e36cda66bc1b 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import copy import functools import inspect import tempfile @@ -939,7 +938,6 @@ def test_save_load_to_base(self): max_diff = (base_params[key] - base_params_from_head[key]).sum().item() self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - # input embeds is meaningless for an encoder-only acoustic model def test_inputs_embeds(self): pass From dc4af2ca3ef9aff84c22a4061d31f395a73696df Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 16 Mar 2023 13:16:19 +0530 Subject: [PATCH 016/277] Fix failing tests --- tests/models/whisper/test_modeling_flax_whisper.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index e36cda66bc1b..1d9ec6c6c19c 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -949,3 +949,12 @@ def test_model_common_attributes(self): # WhisperEncoder cannot resize token embeddings since it has no tokens embeddings def test_resize_tokens_embeddings(self): pass + + # WhisperEncoder does not have any base model + def test_save_load_to_base(self): + pass + + # WhisperEncoder does not have any base model + def test_save_load_from_base(self): + pass + From a55050210ad32fb04556c47f32e16a8ca82e8730 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 16 Mar 2023 13:24:56 +0530 Subject: [PATCH 017/277] Fix failing tests --- .../whisper/test_modeling_flax_whisper.py | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index 1d9ec6c6c19c..9a20c0613283 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -880,29 +880,6 @@ def model_jitted(input_features, decoder_input_ids, **kwargs): self.assertEqual(jitted_output.shape, output.shape) # overwrite because of `input_features` - def test_save_load_from_base(self): - config, _ = self.model_tester.prepare_config_and_inputs_for_common() - base_class = make_partial_class(FLAX_MODEL_MAPPING[config.__class__], input_shape=self.init_shape) - - for model_class in self.all_model_classes: - if model_class.__name__ == base_class.__name__: - continue - - model = base_class(config) - base_params = flatten_dict(unfreeze(model.params)) - - # check that all base model weights are loaded correctly - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - head_model = model_class.from_pretrained(tmpdirname) - - base_param_from_head = flatten_dict(unfreeze(head_model.params[head_model.base_model_prefix])) - # base_param_from_head = get_params(head_model.params, from_head_prefix=head_model.base_model_prefix) - - for key in base_param_from_head.keys(): - max_diff = (base_params[key] - base_param_from_head[key]).sum().item() - self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() @@ -915,30 +892,6 @@ def test_forward_signature(self): expected_arg_names = ["input_features", "attention_mask", "output_attentions"] self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names) - # overwrite because of `input_features` - def test_save_load_to_base(self): - config, _ = self.model_tester.prepare_config_and_inputs_for_common() - base_class = make_partial_class(FLAX_MODEL_MAPPING[config.__class__], input_shape=self.init_shape) - - for model_class in self.all_model_classes: - if model_class.__name__ == base_class.__name__: - continue - - model = model_class(config) - base_params_from_head = flatten_dict(unfreeze(model.params[model.base_model_prefix])) - - # check that all base model weights are loaded correctly - with tempfile.TemporaryDirectory() as tmpdirname: - model.save_pretrained(tmpdirname) - base_model = base_class.from_pretrained(tmpdirname) - - base_params = flatten_dict(unfreeze(base_model.params)) - - for key in base_params_from_head.keys(): - max_diff = (base_params[key] - base_params_from_head[key]).sum().item() - self.assertLessEqual(max_diff, 1e-3, msg=f"{key} not identical") - - # input embeds is meaningless for an encoder-only acoustic model def test_inputs_embeds(self): pass @@ -957,4 +910,3 @@ def test_save_load_to_base(self): # WhisperEncoder does not have any base model def test_save_load_from_base(self): pass - From df8b7625c2bee1f1af28e6b964c6d071df783be2 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 16 Mar 2023 13:52:43 +0530 Subject: [PATCH 018/277] Fix failing tests --- .../whisper/test_modeling_flax_whisper.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index 9a20c0613283..56265adf618a 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -865,8 +865,8 @@ def test_jit_compilation(self): model = model_class(config) @jax.jit - def model_jitted(input_features, decoder_input_ids, **kwargs): - return model(input_features=input_features, decoder_input_ids=decoder_input_ids, **kwargs) + def model_jitted(input_features, **kwargs): + return model(input_features=input_features, **kwargs) with self.subTest("JIT Enabled"): jitted_outputs = model_jitted(**prepared_inputs_dict).to_tuple() @@ -910,3 +910,18 @@ def test_save_load_to_base(self): # WhisperEncoder does not have any base model def test_save_load_from_base(self): pass + + # WhisperEncoder does not have any base model + @is_pt_flax_cross_test + def test_save_load_from_base_pt(self): + pass + + # WhisperEncoder does not have any base model + @is_pt_flax_cross_test + def test_save_load_to_base_pt(self): + pass + + # WhisperEncoder does not have any base model + @is_pt_flax_cross_test + def test_save_load_bf16_to_base_pt(self): + pass From e977c7234c70e88344f9b53a9bfde2b05454d87a Mon Sep 17 00:00:00 2001 From: raghavanone Date: Thu, 16 Mar 2023 15:04:12 +0530 Subject: [PATCH 019/277] Fix tests --- tests/models/whisper/test_modeling_flax_whisper.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index 56265adf618a..d8df00ed0a6d 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -785,12 +785,19 @@ def get_config(self): is_decoder=self.is_decoder, ) + def prepare_whisper_encoder_inputs_dict( + self, + input_features, + ): + return { + "input_features": input_features, + } + def prepare_config_and_inputs(self): input_features = floats_tensor([self.batch_size, self.num_mel_bins, self.seq_length]) config = self.get_config() - inputs_dict = prepare_whisper_encoder_inputs_dict( - config, + inputs_dict = self.prepare_whisper_encoder_inputs_dict( input_features=input_features, ) return config, inputs_dict @@ -843,7 +850,7 @@ class WhisperEncoderModelTest(FlaxModelTesterMixin, unittest.TestCase): # self.maxDiff = 3000 def setUp(self): - self.model_tester = FlaxWhisperModelTester(self) + self.model_tester = FlaxWhisperEncoderModelTester(self) _, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() self.init_shape = (1,) + inputs_dict["input_features"].shape[1:] From e0a39b644c1a93fb961dabd8cf855dc379336bd3 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Sat, 18 Mar 2023 12:47:08 +0530 Subject: [PATCH 020/277] Fix failing tests --- tests/models/whisper/test_modeling_whisper.py | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/tests/models/whisper/test_modeling_whisper.py b/tests/models/whisper/test_modeling_whisper.py index 6fa50a7f0819..4ceb30cfeaa2 100644 --- a/tests/models/whisper/test_modeling_whisper.py +++ b/tests/models/whisper/test_modeling_whisper.py @@ -1561,3 +1561,158 @@ def test_model_common_attributes(self): # WhisperEncoder cannot resize token embeddings since it has no tokens embeddings def test_resize_tokens_embeddings(self): pass + + # FlaxWhisper model have an extra input param 'input_shape' + @is_pt_flax_cross_test + def test_equivalence_flax_to_pt(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + fx_model_class_name = "Flax" + model_class.__name__ + + if not hasattr(transformers, fx_model_class_name): + # no flax model exists for this class + return + + # Output all for aggressive testing + config.output_hidden_states = True + config.output_attentions = self.has_attentions + + fx_model_class = getattr(transformers, fx_model_class_name) + + # load PyTorch class + pt_model = model_class(config).eval() + # Flax models don't use the `use_cache` option and cache is not returned as a default. + # So we disable `use_cache` here for PyTorch model. + pt_model.config.use_cache = False + + # load Flax class + input_shape = (1,) + inputs_dict["input_features"].shape[1:] + fx_model = fx_model_class(config, input_shape=input_shape, dtype=jnp.float32) + + # make sure only flax inputs are forward that actually exist in function args + fx_input_keys = inspect.signature(fx_model.__call__).parameters.keys() + + # prepare inputs + pt_inputs = self._prepare_for_class(inputs_dict, model_class) + + # remove function args that don't exist in Flax + pt_inputs = {k: v for k, v in pt_inputs.items() if k in fx_input_keys} + + # send pytorch inputs to the correct device + pt_inputs = { + k: v.to(device=torch_device) if isinstance(v, torch.Tensor) else v for k, v in pt_inputs.items() + } + + # convert inputs to Flax + fx_inputs = {k: np.array(v) for k, v in pt_inputs.items() if torch.is_tensor(v)} + + pt_model = load_flax_weights_in_pytorch_model(pt_model, fx_model.params) + + # make sure weights are tied in PyTorch + pt_model.tie_weights() + + # send pytorch model to the correct device + pt_model.to(torch_device) + + with torch.no_grad(): + pt_outputs = pt_model(**pt_inputs) + fx_outputs = fx_model(**fx_inputs) + + fx_keys = tuple([k for k, v in fx_outputs.items() if v is not None]) + pt_keys = tuple([k for k, v in pt_outputs.items() if v is not None]) + + self.assertEqual(fx_keys, pt_keys) + self.check_pt_flax_outputs(fx_outputs, pt_outputs, model_class) + + with tempfile.TemporaryDirectory() as tmpdirname: + fx_model.save_pretrained(tmpdirname) + pt_model_loaded = model_class.from_pretrained(tmpdirname, from_flax=True) + + # send pytorch model to the correct device + pt_model_loaded.to(torch_device) + pt_model_loaded.eval() + + with torch.no_grad(): + pt_outputs_loaded = pt_model_loaded(**pt_inputs) + + fx_keys = tuple([k for k, v in fx_outputs.items() if v is not None]) + pt_keys = tuple([k for k, v in pt_outputs_loaded.items() if v is not None]) + + self.assertEqual(fx_keys, pt_keys) + self.check_pt_flax_outputs(fx_outputs, pt_outputs_loaded, model_class) + + # FlaxWhisper model have an extra input param 'input_shape' + @is_pt_flax_cross_test + def test_equivalence_pt_to_flax(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + fx_model_class_name = "Flax" + model_class.__name__ + + if not hasattr(transformers, fx_model_class_name): + # no flax model exists for this class + return + + # Output all for aggressive testing + config.output_hidden_states = True + config.output_attentions = self.has_attentions + + fx_model_class = getattr(transformers, fx_model_class_name) + + # load PyTorch class + pt_model = model_class(config).eval() + # Flax models don't use the `use_cache` option and cache is not returned as a default. + # So we disable `use_cache` here for PyTorch model. + pt_model.config.use_cache = False + + input_shape = (1,) + inputs_dict["input_features"].shape[1:] + # load Flax class + fx_model = fx_model_class(config, input_shape=input_shape, dtype=jnp.float32) + + # make sure only flax inputs are forward that actually exist in function args + fx_input_keys = inspect.signature(fx_model.__call__).parameters.keys() + + # prepare inputs + pt_inputs = self._prepare_for_class(inputs_dict, model_class) + + # remove function args that don't exist in Flax + pt_inputs = {k: v for k, v in pt_inputs.items() if k in fx_input_keys} + + # send pytorch inputs to the correct device + pt_inputs = { + k: v.to(device=torch_device) if isinstance(v, torch.Tensor) else v for k, v in pt_inputs.items() + } + + # convert inputs to Flax + fx_inputs = {k: np.array(v) for k, v in pt_inputs.items() if torch.is_tensor(v)} + + fx_state = convert_pytorch_state_dict_to_flax(pt_model.state_dict(), fx_model) + fx_model.params = fx_state + + # send pytorch model to the correct device + pt_model.to(torch_device) + + with torch.no_grad(): + pt_outputs = pt_model(**pt_inputs) + fx_outputs = fx_model(**fx_inputs) + + fx_keys = tuple([k for k, v in fx_outputs.items() if v is not None]) + pt_keys = tuple([k for k, v in pt_outputs.items() if v is not None]) + + self.assertEqual(fx_keys, pt_keys) + self.check_pt_flax_outputs(fx_outputs, pt_outputs, model_class) + + with tempfile.TemporaryDirectory() as tmpdirname: + pt_model.save_pretrained(tmpdirname) + fx_model_loaded = fx_model_class.from_pretrained(tmpdirname, input_shape=input_shape, from_pt=True) + + fx_outputs_loaded = fx_model_loaded(**fx_inputs) + + fx_keys = tuple([k for k, v in fx_outputs_loaded.items() if v is not None]) + pt_keys = tuple([k for k, v in pt_outputs.items() if v is not None]) + + self.assertEqual(fx_keys, pt_keys) + self.check_pt_flax_outputs(fx_outputs_loaded, pt_outputs, model_class) From 98999e7937f888d4231ec179ab377e7d33921028 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Sat, 25 Mar 2023 12:09:23 +0530 Subject: [PATCH 021/277] Incorporate PR feedbacks --- .../models/whisper/modeling_flax_whisper.py | 59 +++++++------------ .../whisper/test_modeling_flax_whisper.py | 27 --------- 2 files changed, 21 insertions(+), 65 deletions(-) diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index 0b45f2b164f1..7f2fccd6d3ac 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -1474,7 +1474,6 @@ def update_inputs_for_generation(self, model_outputs, model_kwargs): class FlaxWhisperForAudioClassificationModule(nn.Module): config: WhisperConfig dtype: jnp.dtype = jnp.float32 - gradient_checkpointing: bool = False def setup(self) -> None: self.encoder = FlaxWhisperEncoder(config=self.config, dtype=self.dtype) @@ -1482,7 +1481,7 @@ def setup(self) -> None: num_layers = self.config.num_hidden_layers + 1 if self.config.use_weighted_layer_sum: self.layer_weights = jnp.repeat(1 / num_layers, num_layers) - self.projector = nn.Dense(self.config.classifier_proj_size) + self.projector = nn.Dense(self.config.classifier_proj_size, dtype=self.dtype) self.classifier = nn.Dense(self.config.num_labels, dtype=self.dtype) def __call__( @@ -1508,7 +1507,7 @@ def __call__( ) if self.config.use_weighted_layer_sum: - hidden_states = jnp.stack(encoder_outputs, dim=1) + hidden_states = jnp.stack(encoder_outputs, axis=1) norm_weights = jax.nn.softmax(self.layer_weights, axis=-1) hidden_states = jnp.sum(hidden_states * jnp.reshape(norm_weights, [-1, 1, 1]), axis=1) else: @@ -1539,12 +1538,6 @@ def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: Froz input_features = jnp.zeros(input_shape, dtype="f4") input_features = input_features.at[(..., -1)].set(self.config.eos_token_id) - decoder_input_ids = jnp.zeros((input_shape[0], 1), dtype="i4") - jnp.ones_like(decoder_input_ids) - - batch_size, sequence_length = decoder_input_ids.shape - jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) - params_rng, dropout_rng = jax.random.split(rng) rngs = {"params": params_rng, "dropout": dropout_rng} @@ -1576,22 +1569,6 @@ def __call__( dropout_rng: PRNGKey = None, **kwargs, ): - r""" - Returns: - - Example: - - ```python - >>> from transformers import WhisperProcessor, FlaxWhisperForConditionalGeneration - >>> from datasets import load_dataset - - >>> processor = WhisperProcessor.from_pretrained("openai/whisper-tiny.en") - >>> model = FlaxWhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en", from_pt=True) - >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="np") - >>> input_features = inputs.input_features - >>> encoder_outputs = model.encode(input_features=input_features) - ```""" output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -1603,10 +1580,6 @@ def __call__( if dropout_rng is not None: rngs["dropout"] = dropout_rng - def _encoder_forward(module, input_features, **kwargs): - encode_module = module._get_encoder_module() - return encode_module(input_features, **kwargs) - return self.module.apply( {"params": params or self.params}, input_features=jnp.array(input_features, dtype="f4"), @@ -1614,7 +1587,6 @@ def _encoder_forward(module, input_features, **kwargs): output_hidden_states=output_hidden_states, return_dict=return_dict, rngs=rngs, - # method=_encoder_forward, ) @@ -1624,16 +1596,27 @@ def _encoder_forward(module, input_features, **kwargs): Transcription example: ```python - >>> from transformers import WhisperProcessor, FlaxWhisperForAudioClassification + >>> import jax.numpy as jnp + >>> from transformers import AutoFeatureExtractor, FlaxWhisperForAudioClassification >>> from datasets import load_dataset - - >>> processor = WhisperProcessor.from_pretrained("openai/whisper-tiny.en") - >>> model = FlaxWhisperForAudioClassification.from_pretrained("openai/whisper-tiny.en", from_pt=True) - >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") - >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="np") + + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id") + >>> model = FlaxWhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id", from_pt=True) + >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True) + + >>> sample = next(iter(ds)) + + >>> inputs = feature_extractor( + ... sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="np" + ... ) >>> input_features = inputs.input_features - >>> outputs = model(input_features=input_features) - >>> logits = outputs.logit + + >>> logits = model(input_features).logits + + >>> predicted_class_ids = jnp.argmax(logits).item() + >>> predicted_label = model.config.id2label[predicted_class_ids] + >>> predicted_label + 'af_za' ``` """ diff --git a/tests/models/whisper/test_modeling_flax_whisper.py b/tests/models/whisper/test_modeling_flax_whisper.py index d8df00ed0a6d..79a2c51039ac 100644 --- a/tests/models/whisper/test_modeling_flax_whisper.py +++ b/tests/models/whisper/test_modeling_flax_whisper.py @@ -705,15 +705,6 @@ def test_tiny_timestamp_generation(self): self.assertEqual(transcript, EXPECTED_TRANSCRIPT) -def prepare_whisper_encoder_inputs_dict( - config, - input_features, -): - return { - "input_features": input_features, - } - - class FlaxWhisperEncoderModelTester: def __init__( self, @@ -820,19 +811,6 @@ def get_subsampled_output_lengths(self, input_lengths): def encoder_seq_length(self): return self.get_subsampled_output_lengths(self.seq_length) - def create_and_check_model_forward(self, config, inputs_dict, freeze_encoder=False): - model = FlaxWhisperForAudioClassification(config=config) - - if freeze_encoder: - model.freeze_encoder() - - input_features = inputs_dict["input_features"] - - # first forward pass - last_hidden_state = model(input_features).logits - - self.parent.assertTrue(last_hidden_state.shape, (13, 2)) - @require_flax class WhisperEncoderModelTest(FlaxModelTesterMixin, unittest.TestCase): @@ -844,11 +822,6 @@ class WhisperEncoderModelTest(FlaxModelTesterMixin, unittest.TestCase): input_name = "input_features" - # def setUp(self): - # self.model_tester = FlaxWhisperEncoderModelTester(self) - # self.config_tester = ConfigTester(self, config_class=WhisperConfig) - # self.maxDiff = 3000 - def setUp(self): self.model_tester = FlaxWhisperEncoderModelTester(self) _, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() From cea7f4213e3ee1b640e90509051499e0aa872d24 Mon Sep 17 00:00:00 2001 From: raghavanone Date: Sat, 25 Mar 2023 12:30:39 +0530 Subject: [PATCH 022/277] Fix styling changes --- .../models/whisper/modeling_flax_whisper.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/whisper/modeling_flax_whisper.py b/src/transformers/models/whisper/modeling_flax_whisper.py index 7f2fccd6d3ac..e04723272af9 100644 --- a/src/transformers/models/whisper/modeling_flax_whisper.py +++ b/src/transformers/models/whisper/modeling_flax_whisper.py @@ -1599,20 +1599,22 @@ def __call__( >>> import jax.numpy as jnp >>> from transformers import AutoFeatureExtractor, FlaxWhisperForAudioClassification >>> from datasets import load_dataset - + >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id") - >>> model = FlaxWhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id", from_pt=True) + >>> model = FlaxWhisperForAudioClassification.from_pretrained( + ... "sanchit-gandhi/whisper-medium-fleurs-lang-id", from_pt=True + ... ) >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True) - + >>> sample = next(iter(ds)) - + >>> inputs = feature_extractor( ... sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="np" ... ) >>> input_features = inputs.input_features - + >>> logits = model(input_features).logits - + >>> predicted_class_ids = jnp.argmax(logits).item() >>> predicted_label = model.config.id2label[predicted_class_ids] >>> predicted_label From a125b36b0809ba4876760a6e75d57b2bf0d058e1 Mon Sep 17 00:00:00 2001 From: jim <459633157@qq.com> Date: Wed, 8 Mar 2023 19:47:55 +0800 Subject: [PATCH 023/277] Generate - add 1 to cur_len to make up the new beam length (#21993) * add 1 to cur_len to make up the new beam length cur_len is 1 token shorter comparing to the length of the sequence whose best_sum_logprobs is the numerator. * cur_len+=1 before check if beam hyp is done * format code * reformat with black --------- Co-authored-by: Chiming --- src/transformers/generation/beam_search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/generation/beam_search.py b/src/transformers/generation/beam_search.py index d29675f75754..5c423a6a1807 100644 --- a/src/transformers/generation/beam_search.py +++ b/src/transformers/generation/beam_search.py @@ -287,6 +287,7 @@ def process( ) # Check if we are done so that we can save a pad step if all(done) + cur_len += 1 # add up to the length which the next_scores is calculated on self._done[batch_idx] = self._done[batch_idx] or beam_hyp.is_done( next_scores[batch_idx].max().item(), cur_len ) @@ -616,6 +617,7 @@ def process( ) # Check if we are done so that we can save a pad step if all(done) + cur_len += 1 # add up to the length which the next_scores is calculated on self._done[batch_idx] = self._done[batch_idx] or beam_hyp.is_done( next_scores[batch_idx].max().item(), cur_len ) From 4fe0a862e4f15e9606fb4920a3da6f886cede1da Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Wed, 8 Mar 2023 11:54:42 +0000 Subject: [PATCH 024/277] VideoMAE doctest - use valid dummy pixel values (#22022) Use valid dummy pixel values --- src/transformers/models/videomae/modeling_videomae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/videomae/modeling_videomae.py b/src/transformers/models/videomae/modeling_videomae.py index 0d1f12b87308..2ec9c01e6f0f 100644 --- a/src/transformers/models/videomae/modeling_videomae.py +++ b/src/transformers/models/videomae/modeling_videomae.py @@ -790,7 +790,7 @@ def forward( >>> import torch >>> num_frames = 16 - >>> video = list(np.random.randn(16, 3, 224, 224)) + >>> video = list(np.random.randint(0, 256, (num_frames, 3, 224, 224))) >>> image_processor = AutoImageProcessor.from_pretrained("MCG-NJU/videomae-base") >>> model = VideoMAEForPreTraining.from_pretrained("MCG-NJU/videomae-base") From 321ec44c6a163ea30c8959845591fe7f1b61c0b4 Mon Sep 17 00:00:00 2001 From: Qiushi <1530875862@qq.com> Date: Wed, 8 Mar 2023 20:54:30 +0800 Subject: [PATCH 025/277] update: bertology paper (#22012) --- docs/source/en/bertology.mdx | 1 + docs/source/es/bertology.mdx | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/source/en/bertology.mdx b/docs/source/en/bertology.mdx index e64379d6580d..505b56902817 100644 --- a/docs/source/en/bertology.mdx +++ b/docs/source/en/bertology.mdx @@ -21,6 +21,7 @@ There is a growing field of study concerned with investigating the inner working - Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: https://arxiv.org/abs/1905.10650 - What Does BERT Look At? An Analysis of BERT's Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D. Manning: https://arxiv.org/abs/1906.04341 +- CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: https://arxiv.org/abs/2210.04633 In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to help people access the inner representations, mainly adapted from the great work of Paul Michel diff --git a/docs/source/es/bertology.mdx b/docs/source/es/bertology.mdx index 4a3a1e551bcf..9a7c48874256 100644 --- a/docs/source/es/bertology.mdx +++ b/docs/source/es/bertology.mdx @@ -21,6 +21,7 @@ Hay un creciente campo de estudio empeñado en la investigación del funcionamie - Are Sixteen Heads Really Better than One? por Paul Michel, Omer Levy, Graham Neubig: https://arxiv.org/abs/1905.10650 - What Does BERT Look At? An Analysis of BERT's Attention por Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D. Manning: https://arxiv.org/abs/1906.04341 +- CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: https://arxiv.org/abs/2210.04633 Para asistir al desarrollo de este nuevo campo, hemos incluido algunas features adicionales en los modelos BERT/GPT/GPT-2 para ayudar a acceder a las representaciones internas, principalmente adaptado de la gran obra de Paul Michel From 1fd930d8b78fc058d79a529bc67d114ab7aa23c6 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:56:47 +0100 Subject: [PATCH 026/277] Update `AudioClassificationPipelineTests::test_small_model_pt` for PT 2.0.0 (#22023) fix Co-authored-by: ydshieh --- .../test_pipelines_audio_classification.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/pipelines/test_pipelines_audio_classification.py b/tests/pipelines/test_pipelines_audio_classification.py index b0ff55179735..208690396c4a 100644 --- a/tests/pipelines/test_pipelines_audio_classification.py +++ b/tests/pipelines/test_pipelines_audio_classification.py @@ -88,15 +88,20 @@ def test_small_model_pt(self): audio = np.ones((8000,)) output = audio_classifier(audio, top_k=4) - self.assertEqual( - nested_simplify(output, decimals=4), - [ - {"score": 0.0842, "label": "no"}, - {"score": 0.0838, "label": "up"}, - {"score": 0.0837, "label": "go"}, - {"score": 0.0834, "label": "right"}, - ], - ) + + EXPECTED_OUTPUT = [ + {"score": 0.0842, "label": "no"}, + {"score": 0.0838, "label": "up"}, + {"score": 0.0837, "label": "go"}, + {"score": 0.0834, "label": "right"}, + ] + EXPECTED_OUTPUT_PT_2 = [ + {"score": 0.0845, "label": "stop"}, + {"score": 0.0844, "label": "on"}, + {"score": 0.0841, "label": "right"}, + {"score": 0.0834, "label": "left"}, + ] + self.assertIn(nested_simplify(output, decimals=4), [EXPECTED_OUTPUT, EXPECTED_OUTPUT_PT_2]) @require_torch @slow From ee161ed7db5818a9d954b42fb9b5164e454ccdc0 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 8 Mar 2023 14:51:44 +0100 Subject: [PATCH 027/277] [`bnb`] Fix bnb error message (#22026) * fix error message * make style --- src/transformers/modeling_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e7b0aab459f0..494367b813bf 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2578,8 +2578,11 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P raise ValueError( """ Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to fit - the quantized model. If you have set a value for `max_memory` you should increase that. To have - an idea of the modules that are set on the CPU or RAM you can print model.hf_device_map. + the quantized model. If you want to dispatch the model on the CPU or the disk while keeping + these modules in 32-bit, you need to set `load_in_8bit_fp32_cpu_offload=True` and pass a custom + `device_map` to `from_pretrained`. Check + https://huggingface.co/docs/transformers/main/en/main_classes/quantization#offload-between-cpu-and-gpu + for more details. """ ) del device_map_without_lm_head From 2f66a425c45bf142b97c10645ad9395af1675361 Mon Sep 17 00:00:00 2001 From: Anahita Bhiwandiwalla Date: Wed, 8 Mar 2023 06:00:54 -0800 Subject: [PATCH 028/277] [WIP] Add BridgeTowerForContrastiveLearning (#21964) * Add BridgeTower for ITC * Fix review feedback * Rename BridgeTowerForITC, cleanup * Fix style and quality * implement tests --------- Co-authored-by: Tiep Le <97980157+tileintel@users.noreply.github.com> Co-authored-by: Tiep Le --- docs/source/en/model_doc/bridgetower.mdx | 27 +++ src/transformers/__init__.py | 2 + .../models/bridgetower/__init__.py | 2 + .../bridgetower/modeling_bridgetower.py | 187 +++++++++++++++++- src/transformers/utils/dummy_pt_objects.py | 7 + .../bridgetower/test_modeling_bridgetower.py | 76 ++++++- utils/check_repo.py | 1 + 7 files changed, 292 insertions(+), 10 deletions(-) diff --git a/docs/source/en/model_doc/bridgetower.mdx b/docs/source/en/model_doc/bridgetower.mdx index 87015877dc9c..9f7572f3122b 100644 --- a/docs/source/en/model_doc/bridgetower.mdx +++ b/docs/source/en/model_doc/bridgetower.mdx @@ -42,6 +42,28 @@ In principle, one can apply any visual, textual or cross-modal encoder in the pr The [`BridgeTowerProcessor`] wraps [`RobertaTokenizer`] and [`BridgeTowerImageProcessor`] into a single instance to both encode the text and prepare the images respectively. +The following example shows how to run contrastive learning using [`BridgeTowerProcessor`] and [`BridgeTowerForContrastiveLearning`]. +```python +>>> from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning +>>> import requests +>>> from PIL import Image + +>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" +>>> image = Image.open(requests.get(url, stream=True).raw) +>>> texts = ["An image of two cats chilling on a couch", "A football player scoring a goal"] + +>>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") +>>> model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") + +>>> # forward pass +>>> scores = dict() +>>> for text in texts: +... # prepare inputs +... encoding = processor(image, text, return_tensors="pt") +... outputs = model(**encoding) +... scores[text] = outputs +``` + The following example shows how to run image-text retrieval using [`BridgeTowerProcessor`] and [`BridgeTowerForImageAndTextRetrieval`]. ```python >>> from transformers import BridgeTowerProcessor, BridgeTowerForImageAndTextRetrieval @@ -128,6 +150,11 @@ Tips: [[autodoc]] BridgeTowerModel - forward +## BridgeTowerForContrastiveLearning + +[[autodoc]] BridgeTowerForContrastiveLearning + - forward + ## BridgeTowerForMaskedLM [[autodoc]] BridgeTowerForMaskedLM diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 6b16212ee790..19ff7a1f83af 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1182,6 +1182,7 @@ _import_structure["models.bridgetower"].extend( [ "BRIDGETOWER_PRETRAINED_MODEL_ARCHIVE_LIST", + "BridgeTowerForContrastiveLearning", "BridgeTowerForImageAndTextRetrieval", "BridgeTowerForMaskedLM", "BridgeTowerModel", @@ -4667,6 +4668,7 @@ ) from .models.bridgetower import ( BRIDGETOWER_PRETRAINED_MODEL_ARCHIVE_LIST, + BridgeTowerForContrastiveLearning, BridgeTowerForImageAndTextRetrieval, BridgeTowerForMaskedLM, BridgeTowerModel, diff --git a/src/transformers/models/bridgetower/__init__.py b/src/transformers/models/bridgetower/__init__.py index 7058fffa529e..cbd5bd4a366a 100644 --- a/src/transformers/models/bridgetower/__init__.py +++ b/src/transformers/models/bridgetower/__init__.py @@ -42,6 +42,7 @@ else: _import_structure["modeling_bridgetower"] = [ "BRIDGETOWER_PRETRAINED_MODEL_ARCHIVE_LIST", + "BridgeTowerForContrastiveLearning", "BridgeTowerForImageAndTextRetrieval", "BridgeTowerForMaskedLM", "BridgeTowerModel", @@ -74,6 +75,7 @@ else: from .modeling_bridgetower import ( BRIDGETOWER_PRETRAINED_MODEL_ARCHIVE_LIST, + BridgeTowerForContrastiveLearning, BridgeTowerForImageAndTextRetrieval, BridgeTowerForMaskedLM, BridgeTowerModel, diff --git a/src/transformers/models/bridgetower/modeling_bridgetower.py b/src/transformers/models/bridgetower/modeling_bridgetower.py index bae846201098..f405407d7d9b 100644 --- a/src/transformers/models/bridgetower/modeling_bridgetower.py +++ b/src/transformers/models/bridgetower/modeling_bridgetower.py @@ -143,9 +143,8 @@ class BridgeTowerModelOutput(ModelOutput): token), respectively, after further processing through layers used for auxiliary pretraining tasks. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + - one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - - Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of + the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. @@ -161,6 +160,40 @@ class BridgeTowerModelOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None +@dataclass +class BridgeTowerContrastiveOutput(ModelOutput): + """ + Output type of ['BridgeTowerForContrastiveLearning'] + + Args: + logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + text_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`): + The text embeddings obtained by applying the projection layer to the pooler_output. + image_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`): + The image embeddings obtained by applying the projection layer to the pooler_output. + cross_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`): + The text-image cross-modal embeddings obtained by applying the projection layer to the pooler_output. + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Image-text contrastive loss. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of + the model at the output of each layer plus the optional initial embedding outputs. + """ + + logits: torch.FloatTensor = None + text_embeds: Optional[Tuple[torch.FloatTensor]] = None + image_embeds: Optional[Tuple[torch.FloatTensor]] = None + cross_embeds: Optional[Tuple[torch.FloatTensor]] = None + loss: Optional[torch.FloatTensor] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + + class BridgeTowerResidualAttention(nn.Module): def __init__(self, config): super().__init__() @@ -1314,7 +1347,12 @@ def forward( if output_hidden_states: all_hidden_states_text += (text_embeds,) - image_embeds = self.vision_model.visual.forward_pre(pixel_values.type(self.vision_model.dtype)) + if image_embeds is None: + image_embeds = self.vision_model.visual.forward_pre(pixel_values.type(self.vision_model.dtype)) + else: + # Permute as BridgeTowerResidualAttention has batch_first=True + image_embeds = image_embeds.permute(1, 0, 2) + if output_hidden_states: all_hidden_states_image += (image_embeds,) @@ -1438,7 +1476,11 @@ def forward( all_hidden_states = (all_hidden_states_text, all_hidden_states_image, all_hidden_states_cross) if not return_dict: - return tuple(v for v in [text_features, image_features, cls_features] if v is not None) + return tuple( + v + for v in [text_features, image_features, cls_features, all_hidden_states, all_self_attentions] + if v is not None + ) return BridgeTowerModelOutput( text_features=text_features, @@ -1700,3 +1742,138 @@ def forward( hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) + + +class BridgeTowerContrastiveHead(nn.Module): + def __init__(self, hidden_size, embed_size): + super().__init__() + self.fc = nn.Linear(hidden_size, embed_size) + + def forward(self, x): + x = self.fc(x) + return x + + +@add_start_docstrings( + """ + BridgeTower Model with a image-text contrastive head on top computing image-text contrastive loss. + """, + BRIDGETOWER_START_DOCSTRING, +) +class BridgeTowerForContrastiveLearning(BridgeTowerPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.bridgetower = BridgeTowerModel(config) + + self.itc_text_head = BridgeTowerContrastiveHead(config.hidden_size, config.contrastive_hidden_size) + self.itc_image_head = BridgeTowerContrastiveHead(config.hidden_size, config.contrastive_hidden_size) + self.itc_cross_modal_head = BridgeTowerContrastiveHead(config.hidden_size * 2, config.contrastive_hidden_size) + + self.logit_scale = nn.Parameter(torch.ones([]) * self.config.logit_scale_init_value) + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(BRIDGETOWER_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BridgeTowerContrastiveOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + pixel_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + image_embeds: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = True, + return_dict: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + ) -> Union[BridgeTowerContrastiveOutput, Tuple[torch.FloatTensor]]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*): + Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match. + The pairs with 0 will be skipped for calculation. + Returns: + + Examples: + + ```python + >>> from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning + >>> import requests + >>> from PIL import Image + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + >>> texts = "An image of two cats chilling on a couch" + + >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") + >>> model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") + >>> outputs = model(**inputs, output_hidden_states=True) + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.bridgetower( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + pixel_values=pixel_values, + pixel_mask=pixel_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + image_embeds=image_embeds, + output_attentions=output_attentions, + output_hidden_states=True, + return_dict=return_dict, + ) + + pooler_output = outputs.pooler_output if return_dict else outputs[2] + hidden_states_txt, hidden_states_img, hidden_states_cross_modal = ( + outputs.hidden_states if return_dict else outputs[3] + ) + + text_embeds = hidden_states_txt[-1] + image_embeds = hidden_states_img[-1] + + image_embeds_with_ln = self.bridgetower.vision_model.visual.forward_post(image_embeds) + image_token_type_embeddings = self.bridgetower.token_type_embeddings( + torch.full((1,), 1, dtype=torch.long, device=self.bridgetower.token_type_embeddings.weight.device) + ).expand_as(image_embeds_with_ln) + + image_embeds = self.bridgetower.cross_modal_image_transform(image_embeds_with_ln) + image_token_type_embeddings + + # normalized features + text_embeds = nn.functional.normalize(self.itc_text_head(text_embeds[:, 0, :]), dim=-1, p=2) + image_embeds = nn.functional.normalize(self.itc_image_head(image_embeds[:, 0, :]), dim=-1, p=2) + cross_embeds = nn.functional.normalize(self.itc_cross_modal_head(pooler_output), dim=-1, p=2) + + logits = torch.stack([text_embeds, image_embeds, cross_embeds], dim=-2) + + logit_scale = self.logit_scale.exp() + logits_text_to_image = torch.matmul(text_embeds, image_embeds.t()) * logit_scale + logits_text_to_cross = torch.matmul(text_embeds, cross_embeds.t()) * logit_scale + logits_image_to_cross = torch.matmul(image_embeds, cross_embeds.t()) * logit_scale + + itc_loss = None + + if labels is not None: + labels = torch.arange(len(labels), device=logits.device) + text_to_image_loss = nn.functional.cross_entropy(logits_text_to_image, labels) + text_to_cross_loss = nn.functional.cross_entropy(logits_text_to_cross, labels) + image_to_cross_loss = nn.functional.cross_entropy(logits_image_to_cross, labels) + itc_loss = (text_to_image_loss + text_to_cross_loss + image_to_cross_loss) / 3.0 + + if not return_dict: + output = tuple(logits) + return ((itc_loss,) + output) if itc_loss is not None else output + + return BridgeTowerContrastiveOutput( + attentions=outputs.attentions, + hidden_states=outputs.hidden_states, + text_embeds=text_embeds, + image_embeds=image_embeds, + cross_embeds=cross_embeds, + logits=logits, + loss=itc_loss, + ) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index e6323118b7e2..85b4010f38c4 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1328,6 +1328,13 @@ def __init__(self, *args, **kwargs): BRIDGETOWER_PRETRAINED_MODEL_ARCHIVE_LIST = None +class BridgeTowerForContrastiveLearning(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class BridgeTowerForImageAndTextRetrieval(metaclass=DummyObject): _backends = ["torch"] diff --git a/tests/models/bridgetower/test_modeling_bridgetower.py b/tests/models/bridgetower/test_modeling_bridgetower.py index afe3febb6983..9e70c4cdcd44 100644 --- a/tests/models/bridgetower/test_modeling_bridgetower.py +++ b/tests/models/bridgetower/test_modeling_bridgetower.py @@ -24,14 +24,25 @@ from transformers.utils import cached_property from ...test_configuration_common import ConfigTester -from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask +from ...test_modeling_common import ( + ModelTesterMixin, + _config_zero_init, + floats_tensor, + ids_tensor, + random_attention_mask, +) from ...test_pipeline_mixin import PipelineTesterMixin if is_torch_available(): import torch - from transformers import BridgeTowerForImageAndTextRetrieval, BridgeTowerForMaskedLM, BridgeTowerModel + from transformers import ( + BridgeTowerForContrastiveLearning, + BridgeTowerForImageAndTextRetrieval, + BridgeTowerForMaskedLM, + BridgeTowerModel, + ) from transformers.models.bridgetower.modeling_bridgetower import BRIDGETOWER_PRETRAINED_MODEL_ARCHIVE_LIST from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_10 else: @@ -65,6 +76,8 @@ def __init__( text_config=None, vision_config=None, image_size=288, + contrastive_hidden_size=512, + logit_scale_init_value=2.6592, ): self.parent = parent self.share_cross_modal_transformer_layers = share_cross_modal_transformer_layers @@ -90,6 +103,8 @@ def __init__( self.is_training = False self.expected_num_hidden_layers = 32 self.output_hidden_states = output_hidden_states + self.contrastive_hidden_size = contrastive_hidden_size + self.logit_scale_init_value = logit_scale_init_value def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) @@ -118,6 +133,8 @@ def get_config(self): init_layernorm_from_vision_encoder=self.init_layernorm_from_vision_encoder, num_channels=self.num_channels, output_hidden_states=self.output_hidden_states, + contrastive_hidden_size=self.contrastive_hidden_size, + logit_scale_init_value=self.logit_scale_init_value, ) def create_and_check_model( @@ -189,7 +206,14 @@ def prepare_config_and_inputs_for_common(self): @unittest.skipIf(not is_torch_greater_or_equal_than_1_10, "BridgeTower is only available in torch v1.10+") class BridgeTowerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = ( - (BridgeTowerModel, BridgeTowerForImageAndTextRetrieval, BridgeTowerForMaskedLM) if is_torch_available() else () + ( + BridgeTowerModel, + BridgeTowerForImageAndTextRetrieval, + BridgeTowerForMaskedLM, + BridgeTowerForContrastiveLearning, + ) + if is_torch_available() + else () ) pipeline_model_mapping = {"feature-extraction": BridgeTowerModel} if is_torch_available() else {} @@ -347,6 +371,29 @@ def test_retain_grad_hidden_states_attentions(self): if self.has_attentions: self.assertIsNotNone(attentions.grad) + # override as the `logit_scale` parameter initilization is different for BRIDGE TOWER + def test_initialization(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + configs_no_init = _config_zero_init(config) + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + for name, param in model.named_parameters(): + if param.requires_grad: + if name == "logit_scale": + self.assertAlmostEqual( + param.data.item(), + config.logit_scale_init_value, + delta=1e-3, + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + else: + self.assertIn( + ((param.data.mean() * 1e9).round() / 1e9).item(), + [0.0, 1.0], + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + @unittest.skip(reason="""Bridge Tower does not have input/output embeddings. So this test is not applicable.""") def test_model_common_attributes(self): pass @@ -429,12 +476,31 @@ def test_masked_language_modeling(self): outputs = model(**inputs) self.assertAlmostEqual(outputs.loss.item(), 5.7373, places=4) + @slow + def test_constrastive_learning(self): + model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc").to( + torch_device + ) + model.eval() + processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") + image = prepare_img() + text = "a bunch of cats laying on a tower." + inputs = processor(image, text, return_tensors="pt").to(torch_device) + with torch.no_grad(): + outputs = model(**inputs, output_hidden_states=True) + + # verify the logits + expected_shape = torch.Size([1, 3, 512]) + self.assertEqual(outputs.logits.shape, expected_shape) + @require_torch @unittest.skipIf(not is_torch_greater_or_equal_than_1_10, "BridgeTower is only available in torch v1.10+") class BridgeTowerModelTrainingTest(unittest.TestCase): all_training_supported_model_classes = ( - (BridgeTowerForImageAndTextRetrieval, BridgeTowerForMaskedLM) if is_torch_available() else () + (BridgeTowerForImageAndTextRetrieval, BridgeTowerForMaskedLM, BridgeTowerForContrastiveLearning) + if is_torch_available() + else () ) def setUp(self): @@ -445,7 +511,7 @@ def _prepare_inputs_for_training(self, model_class): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() if model_class == BridgeTowerForMaskedLM: inputs_dict["labels"] = inputs_dict["input_ids"] - elif model_class == BridgeTowerForImageAndTextRetrieval: + elif model_class == BridgeTowerForImageAndTextRetrieval or model_class == BridgeTowerForContrastiveLearning: inputs_dict["labels"] = ids_tensor([1], 2) return config, inputs_dict diff --git a/utils/check_repo.py b/utils/check_repo.py index af0237d38c8a..f16c4fb851bf 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -204,6 +204,7 @@ "Swin2SRForImageSuperResolution", "BridgeTowerForImageAndTextRetrieval", "BridgeTowerForMaskedLM", + "BridgeTowerForContrastiveLearning", "CLIPSegForImageSegmentation", "CLIPSegVisionModel", "CLIPSegTextModel", From fe03e5185f7e81d9cd3924adc5ba434514fb250f Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 8 Mar 2023 09:12:43 -0500 Subject: [PATCH 029/277] Fix test for torchneuroncore in Trainer (#22028) --- src/transformers/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 379881f6893c..518e58342afe 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -1538,7 +1538,7 @@ def patched_optimizer_step(optimizer, barrier=False, optimizer_args={}): if self.args.ddp_bucket_cap_mb is not None: kwargs["bucket_cap_mb"] = self.args.ddp_bucket_cap_mb - if is_torch_neuroncore_available: + if is_torch_neuroncore_available(): return model model = nn.parallel.DistributedDataParallel( model, From 781edc5147f8c2bb47466ab87d838d6bd5ec1751 Mon Sep 17 00:00:00 2001 From: anruijian <115125339+anruijian@users.noreply.github.com> Date: Wed, 8 Mar 2023 11:43:31 -0500 Subject: [PATCH 030/277] Add tokenize_kwargs parameter definition in the FeatureExtractionPipeline (#22031) add tokenize_kwargs doc in the FeatureExtractionPipeline --- src/transformers/pipelines/feature_extraction.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/pipelines/feature_extraction.py b/src/transformers/pipelines/feature_extraction.py index f2dc6eaaaeac..b8b5eafeb760 100644 --- a/src/transformers/pipelines/feature_extraction.py +++ b/src/transformers/pipelines/feature_extraction.py @@ -53,6 +53,8 @@ class FeatureExtractionPipeline(Pipeline): device (`int`, *optional*, defaults to -1): Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, a positive will run the model on the associated CUDA device id. + tokenize_kwargs (`dict`, *optional*): + Additional dictionary of keyword arguments passed along to the tokenizer. """ def _sanitize_parameters(self, truncation=None, tokenize_kwargs=None, return_tensors=None, **kwargs): From 57b3a9796ac8a867b1a334e4d01df5d66a196d08 Mon Sep 17 00:00:00 2001 From: bofeng huang Date: Wed, 8 Mar 2023 17:59:31 +0100 Subject: [PATCH 031/277] [examples/speech-recognition] Add SpecAugment to run_speech_recognition_seq2seq.py (#21942) * Add specaugment to run_speech_recognition_seq2seq.py * Remove useless argument: text_column * Fix quality * Update return_attention_mask condition * Update specaugment arguments only for whisper models * Remove SpecAugment arguments from ModelArguments, only leave default values for simplicity * Apply suggestions from code review Co-authored-by: Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> * Update apply_spec_augment only for whisper models * Apply suggestions from code review Co-authored-by: Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> * Rename return_attention_mask to forward_attention_mask to avoid confusion with wav2vec2 models --------- Co-authored-by: Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> --- .../run_speech_recognition_seq2seq.py | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py index c841e99df215..f27f70016288 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py @@ -113,6 +113,12 @@ class ModelArguments: suppress_tokens: List[int] = field( default=None, metadata={"help": "A list of tokens that will be suppressed at generation."} ) + apply_spec_augment: bool = field( + default=False, + metadata={ + "help": "Whether to apply *SpecAugment* data augmentation to the input features. This is currently only relevant for Wav2Vec2, HuBERT, WavLM and Whisper models." + }, + ) @dataclass @@ -127,10 +133,6 @@ class DataTrainingArguments: dataset_config_name: Optional[str] = field( default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} ) - text_column: Optional[str] = field( - default=None, - metadata={"help": "The name of the column in the datasets containing the full texts (for summarization)."}, - ) overwrite_cache: bool = field( default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} ) @@ -227,10 +229,13 @@ class DataCollatorSpeechSeq2SeqWithPadding: The processor used for processing the data. decoder_start_token_id (`int`) The begin-of-sentence of the decoder. + forward_attention_mask (`bool`) + Whether to return attention_mask. """ processor: Any decoder_start_token_id: int + forward_attention_mask: bool def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]: # split inputs and labels since they have to be of different lengths and need @@ -241,6 +246,9 @@ def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> batch = self.processor.feature_extractor.pad(input_features, return_tensors="pt") + if self.forward_attention_mask: + batch["attention_mask"] = torch.LongTensor([feature["attention_mask"] for feature in features]) + labels_batch = self.processor.tokenizer.pad(label_features, return_tensors="pt") # replace padding with -100 to ignore loss correctly @@ -367,6 +375,10 @@ def main(): config.update({"forced_decoder_ids": model_args.forced_decoder_ids, "suppress_tokens": model_args.suppress_tokens}) + # SpecAugment for whisper models + if getattr(config, "model_type", None) == "whisper": + config.update({"apply_spec_augment": model_args.apply_spec_augment}) + feature_extractor = AutoFeatureExtractor.from_pretrained( model_args.feature_extractor_name if model_args.feature_extractor_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, @@ -418,6 +430,12 @@ def main(): text_column_name = data_args.text_column_name model_input_name = feature_extractor.model_input_names[0] do_lower_case = data_args.do_lower_case + # if SpecAugment is used for whisper models, return attention_mask to guide the mask along time axis + forward_attention_mask = ( + getattr(config, "model_type", None) == "whisper" + and getattr(config, "apply_spec_augment", False) + and getattr(config, "mask_time_prob", 0) > 0 + ) if data_args.max_train_samples is not None: raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples)) @@ -428,10 +446,14 @@ def main(): def prepare_dataset(batch): # process audio sample = batch[audio_column_name] - inputs = feature_extractor(sample["array"], sampling_rate=sample["sampling_rate"]) + inputs = feature_extractor( + sample["array"], sampling_rate=sample["sampling_rate"], return_attention_mask=forward_attention_mask + ) # process audio length batch[model_input_name] = inputs.get(model_input_name)[0] batch["input_length"] = len(sample["array"]) + if forward_attention_mask: + batch["attention_mask"] = inputs.get("attention_mask")[0] # process targets input_str = batch[text_column_name].lower() if do_lower_case else batch[text_column_name] @@ -496,6 +518,7 @@ def compute_metrics(pred): data_collator = DataCollatorSpeechSeq2SeqWithPadding( processor=processor, decoder_start_token_id=model.config.decoder_start_token_id, + forward_attention_mask=forward_attention_mask, ) # 11. Initialize Trainer From 9a10d4bbca13f1304be569d08b7b063abed7da38 Mon Sep 17 00:00:00 2001 From: Somasree Majumder <56045049+soma2000-lang@users.noreply.github.com> Date: Thu, 9 Mar 2023 00:51:38 +0530 Subject: [PATCH 032/277] fixes the gradient checkpointing of whisper (#22019) * fixing * Update modeling_whisper.py * Update modeling_whisper.py * Update src/transformers/models/whisper/modeling_whisper.py --------- Co-authored-by: Joao Gante --- src/transformers/models/whisper/modeling_whisper.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index 2e4dfa67f9c7..cefcac389507 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -1029,6 +1029,12 @@ def forward( hidden_states = inputs_embeds + positions hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..." + ) + use_cache = False # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -1053,12 +1059,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning( - "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache =" - " False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 65f56eccb20ef64e371f3e57a95adc54d31b2f59 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 8 Mar 2023 20:27:30 +0100 Subject: [PATCH 033/277] Avoid `text_config_dict` and `vision_config_dict` being saved for CLIP-like models (#22035) * Avoid text_config_dict and vision_config_dict being saved * for other CLIP-like models --------- Co-authored-by: ydshieh --- .../models/altclip/configuration_altclip.py | 72 ++++++++- .../models/blip/configuration_blip.py | 12 +- .../bridgetower/configuration_bridgetower.py | 15 +- .../configuration_chinese_clip.py | 72 ++++++++- .../models/clip/configuration_clip.py | 72 ++++++++- .../models/clipseg/configuration_clipseg.py | 73 ++++++++- .../models/flava/configuration_flava.py | 143 ++++++++++++++++-- .../models/groupvit/configuration_groupvit.py | 72 ++++++++- .../models/x_clip/configuration_x_clip.py | 72 ++++++++- 9 files changed, 537 insertions(+), 66 deletions(-) diff --git a/src/transformers/models/altclip/configuration_altclip.py b/src/transformers/models/altclip/configuration_altclip.py index 523cf420e0ae..4ddbb5ec8160 100755 --- a/src/transformers/models/altclip/configuration_altclip.py +++ b/src/transformers/models/altclip/configuration_altclip.py @@ -294,23 +294,83 @@ class AltCLIPConfig(PretrainedConfig): def __init__( self, text_config=None, vision_config=None, projection_dim=768, logit_scale_init_value=2.6592, **kwargs ): - super().__init__(**kwargs) - # If `_config_dict` exist, we use them for the backward compatibility. + # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot + # of confusion!). text_config_dict = kwargs.pop("text_config_dict", None) vision_config_dict = kwargs.pop("vision_config_dict", None) + + super().__init__(**kwargs) + + # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in + # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most + # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. if text_config_dict is not None: - text_config = text_config_dict + if text_config is None: + text_config = {} + + # This is the complete result when using `text_config_dict`. + _text_config_dict = AltCLIPTextConfig(**text_config_dict).to_dict() + + # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. + for key, value in _text_config_dict.items(): + if key in text_config and value != text_config[key] and key not in ["transformers_version"]: + # If specified in `text_config_dict` + if key in text_config_dict: + message = ( + f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " + f'The value `text_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`text_config_dict` is provided which will be used to initialize `AltCLIPTextConfig`. The " + f'value `text_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `text_config` with the ones in `_text_config_dict`. + text_config.update(_text_config_dict) + if vision_config_dict is not None: - vision_config = vision_config_dict + if vision_config is None: + vision_config = {} + + # This is the complete result when using `vision_config_dict`. + _vision_config_dict = AltCLIPVisionConfig(**vision_config_dict).to_dict() + # convert keys to string instead of integer + if "id2label" in _vision_config_dict: + _vision_config_dict["id2label"] = { + str(key): value for key, value in _vision_config_dict["id2label"].items() + } + + # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different. + for key, value in _vision_config_dict.items(): + if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]: + # If specified in `vision_config_dict` + if key in vision_config_dict: + message = ( + f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different " + f'values. The value `vision_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`vision_config_dict` is provided which will be used to initialize `AltCLIPVisionConfig`. " + f'The value `vision_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `vision_config` with the ones in `_vision_config_dict`. + vision_config.update(_vision_config_dict) if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the AltCLIPTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the AltCLIPVisionConfig with default values.") + logger.info("`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values.") self.text_config = AltCLIPTextConfig(**text_config) self.vision_config = AltCLIPVisionConfig(**vision_config) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 8bdff88bff2f..f03f167c2941 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -336,21 +336,13 @@ def __init__( ): super().__init__(**kwargs) - # If `_config_dict` exist, we use them for the backward compatibility. - text_config_dict = kwargs.pop("text_config_dict", None) - vision_config_dict = kwargs.pop("vision_config_dict", None) - if text_config_dict is not None: - text_config = text_config_dict - if vision_config_dict is not None: - vision_config = vision_config_dict - if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the BlipTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `BlipTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the BlipVisionConfig with default values.") + logger.info("`vision_config` is `None`. Initializing the `BlipVisionConfig` with default values.") self.text_config = BlipTextConfig(**text_config) self.vision_config = BlipVisionConfig(**vision_config) diff --git a/src/transformers/models/bridgetower/configuration_bridgetower.py b/src/transformers/models/bridgetower/configuration_bridgetower.py index c04b01a2ea71..3149b34efaac 100644 --- a/src/transformers/models/bridgetower/configuration_bridgetower.py +++ b/src/transformers/models/bridgetower/configuration_bridgetower.py @@ -319,6 +319,10 @@ def __init__( vision_config=None, **kwargs, ): + # TODO: remove this once the Hub files are updated. + _ = kwargs.pop("text_config_dict", None) + _ = kwargs.pop("vision_config_dict", None) + super().__init__(**kwargs) self.share_cross_modal_transformer_layers = share_cross_modal_transformer_layers self.hidden_act = hidden_act @@ -332,20 +336,13 @@ def __init__( self.tie_word_embeddings = tie_word_embeddings self.init_layernorm_from_vision_encoder = init_layernorm_from_vision_encoder - text_config_dict = kwargs.pop("text_config_dict", None) - vision_config_dict = kwargs.pop("vision_config_dict", None) - if text_config_dict is not None: - text_config = text_config_dict - if vision_config_dict is not None: - vision_config = vision_config_dict - if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the BridgeTowerTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `BridgeTowerTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. Initializing the BridgeTowerVisionConfig with default values.") + logger.info("`vision_config` is `None`. Initializing the `BridgeTowerVisionConfig` with default values.") self.text_config = BridgeTowerTextConfig(**text_config) self.vision_config = BridgeTowerVisionConfig(**vision_config) diff --git a/src/transformers/models/chinese_clip/configuration_chinese_clip.py b/src/transformers/models/chinese_clip/configuration_chinese_clip.py index f20e16e41cac..85dfd0e45675 100644 --- a/src/transformers/models/chinese_clip/configuration_chinese_clip.py +++ b/src/transformers/models/chinese_clip/configuration_chinese_clip.py @@ -315,23 +315,83 @@ class ChineseCLIPConfig(PretrainedConfig): def __init__( self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs ): - super().__init__(**kwargs) - # If `_config_dict` exist, we use them for the backward compatibility. + # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot + # of confusion!). text_config_dict = kwargs.pop("text_config_dict", None) vision_config_dict = kwargs.pop("vision_config_dict", None) + + super().__init__(**kwargs) + + # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in + # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most + # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. if text_config_dict is not None: - text_config = text_config_dict + if text_config is None: + text_config = {} + + # This is the complete result when using `text_config_dict`. + _text_config_dict = ChineseCLIPTextConfig(**text_config_dict).to_dict() + + # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. + for key, value in _text_config_dict.items(): + if key in text_config and value != text_config[key] and key not in ["transformers_version"]: + # If specified in `text_config_dict` + if key in text_config_dict: + message = ( + f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " + f'The value `text_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`text_config_dict` is provided which will be used to initialize `ChineseCLIPTextConfig`. " + f'The value `text_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `text_config` with the ones in `_text_config_dict`. + text_config.update(_text_config_dict) + if vision_config_dict is not None: - vision_config = vision_config_dict + if vision_config is None: + vision_config = {} + + # This is the complete result when using `vision_config_dict`. + _vision_config_dict = ChineseCLIPVisionConfig(**vision_config_dict).to_dict() + # convert keys to string instead of integer + if "id2label" in _vision_config_dict: + _vision_config_dict["id2label"] = { + str(key): value for key, value in _vision_config_dict["id2label"].items() + } + + # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different. + for key, value in _vision_config_dict.items(): + if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]: + # If specified in `vision_config_dict` + if key in vision_config_dict: + message = ( + f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different " + f'values. The value `vision_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`vision_config_dict` is provided which will be used to initialize " + f'`ChineseCLIPVisionConfig`. The value `vision_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `vision_config` with the ones in `_vision_config_dict`. + vision_config.update(_vision_config_dict) if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the ChineseCLIPTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `ChineseCLIPTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the ChineseCLIPVisionConfig with default values.") + logger.info("`vision_config` is `None`. initializing the `ChineseCLIPVisionConfig` with default values.") self.text_config = ChineseCLIPTextConfig(**text_config) self.vision_config = ChineseCLIPVisionConfig(**vision_config) diff --git a/src/transformers/models/clip/configuration_clip.py b/src/transformers/models/clip/configuration_clip.py index 129524599311..2485e6893306 100644 --- a/src/transformers/models/clip/configuration_clip.py +++ b/src/transformers/models/clip/configuration_clip.py @@ -297,23 +297,83 @@ class CLIPConfig(PretrainedConfig): def __init__( self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs ): - super().__init__(**kwargs) - # If `_config_dict` exist, we use them for the backward compatibility. + # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot + # of confusion!). text_config_dict = kwargs.pop("text_config_dict", None) vision_config_dict = kwargs.pop("vision_config_dict", None) + + super().__init__(**kwargs) + + # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in + # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most + # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. if text_config_dict is not None: - text_config = text_config_dict + if text_config is None: + text_config = {} + + # This is the complete result when using `text_config_dict`. + _text_config_dict = CLIPTextConfig(**text_config_dict).to_dict() + + # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. + for key, value in _text_config_dict.items(): + if key in text_config and value != text_config[key] and key not in ["transformers_version"]: + # If specified in `text_config_dict` + if key in text_config_dict: + message = ( + f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " + f'The value `text_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The " + f'value `text_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `text_config` with the ones in `_text_config_dict`. + text_config.update(_text_config_dict) + if vision_config_dict is not None: - vision_config = vision_config_dict + if vision_config is None: + vision_config = {} + + # This is the complete result when using `vision_config_dict`. + _vision_config_dict = CLIPVisionConfig(**vision_config_dict).to_dict() + # convert keys to string instead of integer + if "id2label" in _vision_config_dict: + _vision_config_dict["id2label"] = { + str(key): value for key, value in _vision_config_dict["id2label"].items() + } + + # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different. + for key, value in _vision_config_dict.items(): + if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]: + # If specified in `vision_config_dict` + if key in vision_config_dict: + message = ( + f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different " + f'values. The value `vision_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. " + f'The value `vision_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `vision_config` with the ones in `_vision_config_dict`. + vision_config.update(_vision_config_dict) if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the CLIPTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `CLIPTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the CLIPVisionConfig with default values.") + logger.info("`vision_config` is `None`. initializing the `CLIPVisionConfig` with default values.") self.text_config = CLIPTextConfig(**text_config) self.vision_config = CLIPVisionConfig(**vision_config) diff --git a/src/transformers/models/clipseg/configuration_clipseg.py b/src/transformers/models/clipseg/configuration_clipseg.py index 1910c946325a..2bb7360d5c64 100644 --- a/src/transformers/models/clipseg/configuration_clipseg.py +++ b/src/transformers/models/clipseg/configuration_clipseg.py @@ -316,22 +316,83 @@ def __init__( use_complex_transposed_convolution=False, **kwargs, ): - super().__init__(**kwargs) - + # If `_config_dict` exist, we use them for the backward compatibility. + # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot + # of confusion!). text_config_dict = kwargs.pop("text_config_dict", None) vision_config_dict = kwargs.pop("vision_config_dict", None) + + super().__init__(**kwargs) + + # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in + # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most + # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. if text_config_dict is not None: - text_config = text_config_dict + if text_config is None: + text_config = {} + + # This is the complete result when using `text_config_dict`. + _text_config_dict = CLIPSegTextConfig(**text_config_dict).to_dict() + + # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. + for key, value in _text_config_dict.items(): + if key in text_config and value != text_config[key] and key not in ["transformers_version"]: + # If specified in `text_config_dict` + if key in text_config_dict: + message = ( + f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " + f'The value `text_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`text_config_dict` is provided which will be used to initialize `CLIPSegTextConfig`. The " + f'value `text_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `text_config` with the ones in `_text_config_dict`. + text_config.update(_text_config_dict) + if vision_config_dict is not None: - vision_config = vision_config_dict + if vision_config is None: + vision_config = {} + + # This is the complete result when using `vision_config_dict`. + _vision_config_dict = CLIPSegVisionConfig(**vision_config_dict).to_dict() + # convert keys to string instead of integer + if "id2label" in _vision_config_dict: + _vision_config_dict["id2label"] = { + str(key): value for key, value in _vision_config_dict["id2label"].items() + } + + # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different. + for key, value in _vision_config_dict.items(): + if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]: + # If specified in `vision_config_dict` + if key in vision_config_dict: + message = ( + f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different " + f'values. The value `vision_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`vision_config_dict` is provided which will be used to initialize `CLIPSegVisionConfig`. " + f'The value `vision_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `vision_config` with the ones in `_vision_config_dict`. + vision_config.update(_vision_config_dict) if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the CLIPSegTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `CLIPSegTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the CLIPSegVisionConfig with default values.") + logger.info("`vision_config` is `None`. initializing the `CLIPSegVisionConfig` with default values.") self.text_config = CLIPSegTextConfig(**text_config) self.vision_config = CLIPSegVisionConfig(**vision_config) diff --git a/src/transformers/models/flava/configuration_flava.py b/src/transformers/models/flava/configuration_flava.py index da2c9cc95cfb..f5f69ff6e5dc 100644 --- a/src/transformers/models/flava/configuration_flava.py +++ b/src/transformers/models/flava/configuration_flava.py @@ -554,38 +554,159 @@ def __init__( return_loss: bool = True, **kwargs, ): - super().__init__(**kwargs) - # If `_config_dict` exist, we use them for the backward compatibility. + # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot + # of confusion!). text_config_dict = kwargs.pop("text_config_dict", None) - image_config_dict = kwargs.pop("vision_config_dict", None) + image_config_dict = kwargs.pop("image_config_dict", None) multimodal_config_dict = kwargs.pop("multimodal_config_dict", None) image_codebook_config_dict = kwargs.pop("image_codebook_config_dict", None) + + super().__init__(**kwargs) + + # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in + # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most + # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. if text_config_dict is not None: - text_config = text_config_dict + if text_config is None: + text_config = {} + + # This is the complete result when using `text_config_dict`. + _text_config_dict = FlavaTextConfig(**text_config_dict).to_dict() + + # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. + for key, value in _text_config_dict.items(): + if key in text_config and value != text_config[key] and key not in ["transformers_version"]: + # If specified in `text_config_dict` + if key in text_config_dict: + message = ( + f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " + f'The value `text_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`text_config_dict` is provided which will be used to initialize `FlavaTextConfig`. The " + f'value `text_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `text_config` with the ones in `_text_config_dict`. + text_config.update(_text_config_dict) + if image_config_dict is not None: - image_config = image_config_dict + if image_config is None: + image_config = {} + + # This is the complete result when using `image_config_dict`. + _image_config_dict = FlavaImageConfig(**image_config_dict).to_dict() + # convert keys to string instead of integer + if "id2label" in _image_config_dict: + _image_config_dict["id2label"] = { + str(key): value for key, value in _image_config_dict["id2label"].items() + } + + # Give a warning if the values exist in both `_image_config_dict` and `image_config` but being different. + for key, value in _image_config_dict.items(): + if key in image_config and value != image_config[key] and key not in ["transformers_version"]: + # If specified in `image_config_dict` + if key in image_config_dict: + message = ( + f"`{key}` is found in both `image_config_dict` and `image_config` but with different " + f'values. The value `image_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`image_config_dict` is provided which will be used to initialize `FlavaImageConfig`. " + f'The value `image_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `image_config` with the ones in `_image_config_dict`. + image_config.update(_image_config_dict) + if multimodal_config_dict is not None: - multimodal_config = multimodal_config_dict + if multimodal_config is None: + multimodal_config = {} + + # This is the complete result when using `multimodal_config_dict`. + _multimodal_config_dict = FlavaMultimodalConfig(**multimodal_config_dict).to_dict() + + # Give a warning if the values exist in both `_multimodal_config_dict` and `multimodal_config` but being + # different. + for key, value in _multimodal_config_dict.items(): + if ( + key in multimodal_config + and value != multimodal_config[key] + and key not in ["transformers_version"] + ): + # If specified in `multimodal_config_dict` + if key in multimodal_config_dict: + message = ( + f"`{key}` is found in both `multimodal_config_dict` and `multimodal_config` but with " + f'different values. The value `multimodal_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`multimodal_config_dict` is provided which will be used to initialize " + f'`FlavaMultimodalConfig`. The value `multimodal_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `multimodal_config` with the ones in `_multimodal_config_dict`. + multimodal_config.update(_multimodal_config_dict) + if image_codebook_config_dict is not None: - image_codebook_config = image_codebook_config_dict + if image_codebook_config is None: + image_codebook_config = {} + + # This is the complete result when using `image_codebook_config_dict`. + _image_codebook_config_dict = FlavaImageCodebookConfig(**image_codebook_config_dict).to_dict() + + # Give a warning if the values exist in both `_image_codebook_config_dict` and `image_codebook_config` but + # being different. + for key, value in _image_codebook_config_dict.items(): + if ( + key in image_codebook_config + and value != image_codebook_config[key] + and key not in ["transformers_version"] + ): + # If specified in `image_codebook_config_dict` + if key in image_codebook_config_dict: + message = ( + f"`{key}` is found in both `image_codebook_config_dict` and `image_codebook_config` but " + f'with different values. The value `image_codebook_config_dict["{key}"]` will be used ' + "instead." + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`image_codebook_config_dict` is provided which will be used to initialize " + f'`FlavaImageCodebookConfig`. The value `image_codebook_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `image_codebook_config` with the ones in `_image_codebook_config_dict`. + image_codebook_config.update(_image_codebook_config_dict) if image_config is None: image_config = {} - logger.info("image_config is None. initializing the FlavaImageConfig with default values.") + logger.info("`image_config` is `None`. initializing the `FlavaImageConfig` with default values.") if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the FlavaTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `FlavaTextConfig` with default values.") if multimodal_config is None: multimodal_config = {} - logger.info("multimodal_config is None. initializing the FlavaMultimodalConfig with default values.") + logger.info("`multimodal_config` is `None`. initializing the `FlavaMultimodalConfig` with default values.") if image_codebook_config is None: image_codebook_config = {} logger.info( - "image_codebook_config is None. initializing the FlavaImageCodebookConfig with default values." + "`image_codebook_config` is `None`. initializing the `FlavaImageCodebookConfig` with default values." ) self.image_config = FlavaImageConfig(**image_config) diff --git a/src/transformers/models/groupvit/configuration_groupvit.py b/src/transformers/models/groupvit/configuration_groupvit.py index ded75904c555..31fb19cac0de 100644 --- a/src/transformers/models/groupvit/configuration_groupvit.py +++ b/src/transformers/models/groupvit/configuration_groupvit.py @@ -303,23 +303,83 @@ def __init__( logit_scale_init_value=2.6592, **kwargs, ): - super().__init__(**kwargs) - # If `_config_dict` exist, we use them for the backward compatibility. + # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot + # of confusion!). text_config_dict = kwargs.pop("text_config_dict", None) vision_config_dict = kwargs.pop("vision_config_dict", None) + + super().__init__(**kwargs) + + # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in + # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most + # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. if text_config_dict is not None: - text_config = text_config_dict + if text_config is None: + text_config = {} + + # This is the complete result when using `text_config_dict`. + _text_config_dict = GroupViTTextConfig(**text_config_dict).to_dict() + + # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. + for key, value in _text_config_dict.items(): + if key in text_config and value != text_config[key] and key not in ["transformers_version"]: + # If specified in `text_config_dict` + if key in text_config_dict: + message = ( + f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " + f'The value `text_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`text_config_dict` is provided which will be used to initialize `GroupViTTextConfig`. " + f'The value `text_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `text_config` with the ones in `_text_config_dict`. + text_config.update(_text_config_dict) + if vision_config_dict is not None: - vision_config = vision_config_dict + if vision_config is None: + vision_config = {} + + # This is the complete result when using `vision_config_dict`. + _vision_config_dict = GroupViTVisionConfig(**vision_config_dict).to_dict() + # convert keys to string instead of integer + if "id2label" in _vision_config_dict: + _vision_config_dict["id2label"] = { + str(key): value for key, value in _vision_config_dict["id2label"].items() + } + + # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different. + for key, value in _vision_config_dict.items(): + if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]: + # If specified in `vision_config_dict` + if key in vision_config_dict: + message = ( + f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different " + f'values. The value `vision_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`vision_config_dict` is provided which will be used to initialize `GroupViTVisionConfig`." + f' The value `vision_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `vision_config` with the ones in `_vision_config_dict`. + vision_config.update(_vision_config_dict) if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the GroupViTTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `GroupViTTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the GroupViTVisionConfig with default values.") + logger.info("`vision_config` is `None`. initializing the `GroupViTVisionConfig` with default values.") self.text_config = GroupViTTextConfig(**text_config) self.vision_config = GroupViTVisionConfig(**vision_config) diff --git a/src/transformers/models/x_clip/configuration_x_clip.py b/src/transformers/models/x_clip/configuration_x_clip.py index ed2bdea69a50..ddb30d2ad2be 100644 --- a/src/transformers/models/x_clip/configuration_x_clip.py +++ b/src/transformers/models/x_clip/configuration_x_clip.py @@ -311,23 +311,83 @@ def __init__( logit_scale_init_value=2.6592, **kwargs, ): - super().__init__(**kwargs) - # If `_config_dict` exist, we use them for the backward compatibility. + # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot + # of confusion!). text_config_dict = kwargs.pop("text_config_dict", None) vision_config_dict = kwargs.pop("vision_config_dict", None) + + super().__init__(**kwargs) + + # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in + # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most + # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. if text_config_dict is not None: - text_config = text_config_dict + if text_config is None: + text_config = {} + + # This is the complete result when using `text_config_dict`. + _text_config_dict = XCLIPTextConfig(**text_config_dict).to_dict() + + # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. + for key, value in _text_config_dict.items(): + if key in text_config and value != text_config[key] and key not in ["transformers_version"]: + # If specified in `text_config_dict` + if key in text_config_dict: + message = ( + f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " + f'The value `text_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`text_config_dict` is provided which will be used to initialize `XCLIPTextConfig`. The " + f'value `text_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `text_config` with the ones in `_text_config_dict`. + text_config.update(_text_config_dict) + if vision_config_dict is not None: - vision_config = vision_config_dict + if vision_config is None: + vision_config = {} + + # This is the complete result when using `vision_config_dict`. + _vision_config_dict = XCLIPVisionConfig(**vision_config_dict).to_dict() + # convert keys to string instead of integer + if "id2label" in _vision_config_dict: + _vision_config_dict["id2label"] = { + str(key): value for key, value in _vision_config_dict["id2label"].items() + } + + # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different. + for key, value in _vision_config_dict.items(): + if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]: + # If specified in `vision_config_dict` + if key in vision_config_dict: + message = ( + f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different " + f'values. The value `vision_config_dict["{key}"]` will be used instead.' + ) + # If inferred from default argument values (just to be super careful) + else: + message = ( + f"`vision_config_dict` is provided which will be used to initialize `XCLIPVisionConfig`. " + f'The value `vision_config["{key}"]` will be overriden.' + ) + logger.warning(message) + + # Update all values in `vision_config` with the ones in `_vision_config_dict`. + vision_config.update(_vision_config_dict) if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the XCLIPTextConfig with default values.") + logger.info("`text_config` is `None`. Initializing the `XCLIPTextConfig` with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the XCLIPVisionConfig with default values.") + logger.info("`vision_config` is `None`. initializing the `XCLIPVisionConfig` with default values.") self.text_config = XCLIPTextConfig(**text_config) self.vision_config = XCLIPVisionConfig(**vision_config) From 3f29db287851d29fdfbd72242c666283b7d1e904 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 8 Mar 2023 21:48:29 +0100 Subject: [PATCH 034/277] Mark all `BridgeTower` tests slow for now (#22039) * slow me --------- Co-authored-by: ydshieh --- tests/models/bridgetower/test_modeling_bridgetower.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/models/bridgetower/test_modeling_bridgetower.py b/tests/models/bridgetower/test_modeling_bridgetower.py index 9e70c4cdcd44..20396c8bf7bf 100644 --- a/tests/models/bridgetower/test_modeling_bridgetower.py +++ b/tests/models/bridgetower/test_modeling_bridgetower.py @@ -202,6 +202,7 @@ def prepare_config_and_inputs_for_common(self): return config, inputs_dict +@slow @require_torch @unittest.skipIf(not is_torch_greater_or_equal_than_1_10, "BridgeTower is only available in torch v1.10+") class BridgeTowerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): @@ -494,6 +495,7 @@ def test_constrastive_learning(self): self.assertEqual(outputs.logits.shape, expected_shape) +@slow @require_torch @unittest.skipIf(not is_torch_greater_or_equal_than_1_10, "BridgeTower is only available in torch v1.10+") class BridgeTowerModelTrainingTest(unittest.TestCase): From 2a5f185911ac81b36036a43ed6a41321fe12e6ab Mon Sep 17 00:00:00 2001 From: Ceyda Cinarel <15624271+cceyda@users.noreply.github.com> Date: Thu, 9 Mar 2023 06:21:46 +0900 Subject: [PATCH 035/277] Bug fix: token classification pipeline while passing offset_mapping (#22034) fix slow tokenizers with passing offset_mapping --- src/transformers/pipelines/token_classification.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/pipelines/token_classification.py b/src/transformers/pipelines/token_classification.py index 5dc243d1acf2..f3c78d3498d5 100644 --- a/src/transformers/pipelines/token_classification.py +++ b/src/transformers/pipelines/token_classification.py @@ -304,7 +304,9 @@ def gather_pre_entities( start_ind = start_ind.item() end_ind = end_ind.item() word_ref = sentence[start_ind:end_ind] - if getattr(self.tokenizer._tokenizer.model, "continuing_subword_prefix", None): + if getattr(self.tokenizer, "_tokenizer", None) and getattr( + self.tokenizer._tokenizer.model, "continuing_subword_prefix", None + ): # This is a BPE, word aware tokenizer, there is a correct way # to fuse tokens is_subword = len(word) != len(word_ref) From f89b95c16a461e9041e3e91c57b30d4b2f5b93f1 Mon Sep 17 00:00:00 2001 From: Alara Dirik <8944735+alaradirik@users.noreply.github.com> Date: Thu, 9 Mar 2023 14:12:17 +0300 Subject: [PATCH 036/277] Update ALIGN docs (#22025) * Fix typos and add code examples, resources --- docs/source/en/model_doc/align.mdx | 44 ++++++++++++++++++- .../models/align/modeling_align.py | 4 +- .../models/align/processing_align.py | 2 +- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/source/en/model_doc/align.mdx b/docs/source/en/model_doc/align.mdx index 5ffec6bebcdb..043de683c17a 100644 --- a/docs/source/en/model_doc/align.mdx +++ b/docs/source/en/model_doc/align.mdx @@ -14,15 +14,57 @@ specific language governing permissions and limitations under the License. ## Overview -The ALIGN model was proposed in [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. ALIGN features a dual-encoder architecture with [EfficientNet](efficientnet) as its vision encoder and [BERT](bert) as its text encoder, and learns to align visual and text representations with contrastive learning. Unlike previous work, ALIGN leverages a massive noisy dataset and shows that the scale of the corpus can be used to achieve SOTA representations with a simple recipe. +The ALIGN model was proposed in [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. ALIGN is a multi-modal vision and language model. It can be used for image-text similarity and for zero-shot image classification. ALIGN features a dual-encoder architecture with [EfficientNet](efficientnet) as its vision encoder and [BERT](bert) as its text encoder, and learns to align visual and text representations with contrastive learning. Unlike previous work, ALIGN leverages a massive noisy dataset and shows that the scale of the corpus can be used to achieve SOTA representations with a simple recipe. The abstract from the paper is the following: *Pre-trained representations are becoming crucial for many NLP and perception tasks. While representation learning in NLP has transitioned to training on raw text without human annotations, visual and vision-language representations still rely heavily on curated training datasets that are expensive or require expert knowledge. For vision applications, representations are mostly learned using datasets with explicit class labels such as ImageNet or OpenImages. For vision-language, popular datasets like Conceptual Captions, MSCOCO, or CLIP all involve a non-trivial data collection (and cleaning) process. This costly curation process limits the size of datasets and hence hinders the scaling of trained models. In this paper, we leverage a noisy dataset of over one billion image alt-text pairs, obtained without expensive filtering or post-processing steps in the Conceptual Captions dataset. A simple dual-encoder architecture learns to align visual and language representations of the image and text pairs using a contrastive loss. We show that the scale of our corpus can make up for its noise and leads to state-of-the-art representations even with such a simple learning scheme. Our visual representation achieves strong performance when transferred to classification tasks such as ImageNet and VTAB. The aligned visual and language representations enables zero-shot image classification and also set new state-of-the-art results on Flickr30K and MSCOCO image-text retrieval benchmarks, even when compared with more sophisticated cross-attention models. The representations also enable cross-modality search with complex text and text + image queries.* +## Usage + +ALIGN uses EfficientNet to get visual features and BERT to get the text features. Both the text and visual features are then projected to a latent space with identical dimension. The dot product between the projected image and text features is then used as a similarity score. + +[`AlignProcessor`] wraps [`EfficientNetImageProcessor`] and [`BertTokenizer`] into a single instance to both encode the text and preprocess the images. The following example shows how to get the image-text similarity scores using [`AlignProcessor`] and [`AlignModel`]. + +```python +import requests +import torch +from PIL import Image +from transformers import AlignProcessor, AlignModel + +processor = AlignProcessor.from_pretrained("kakaobrain/align-base") +model = AlignModel.from_pretrained("kakaobrain/align-base") + +url = "http://images.cocodataset.org/val2017/000000039769.jpg" +image = Image.open(requests.get(url, stream=True).raw) +candidate_labels = ["an image of a cat", "an image of a dog"] + +inputs = processor(text=candidate_labels, images=image, return_tensors="pt") + +with torch.no_grad(): + outputs = model(**inputs) + +# this is the image-text similarity score +logits_per_image = outputs.logits_per_image + +# we can take the softmax to get the label probabilities +probs = logits_per_image.softmax(dim=1) +print(probs) +``` + This model was contributed by [Alara Dirik](https://huggingface.co/adirik). The original code is not released, this implementation is based on the Kakao Brain implementation based on the original paper. +## Resources + +A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with ALIGN. + +- A blog post on [ALIGN and the COYO-700M dataset](https://huggingface.co/blog). +- A zero-shot image classification [demo](https://huggingface.co/spaces/adirik/ALIGN-zero-shot-image-classification). +- [Model card](https://huggingface.co/kakaobrain/align-base) of `kakaobrain/align-base` model. + +If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we will review it. The resource should ideally demonstrate something new instead of duplicating an existing resource. + ## AlignConfig diff --git a/src/transformers/models/align/modeling_align.py b/src/transformers/models/align/modeling_align.py index 6e74c8205d73..09ee6eca6265 100644 --- a/src/transformers/models/align/modeling_align.py +++ b/src/transformers/models/align/modeling_align.py @@ -122,7 +122,7 @@ Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`EfficientNetImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. @@ -172,7 +172,7 @@ model's internal embedding lookup matrix. pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details. + [`AutoImageProcessor`]. See [`EfficientNetImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): diff --git a/src/transformers/models/align/processing_align.py b/src/transformers/models/align/processing_align.py index 0a26aaa379a3..0863c11310e3 100644 --- a/src/transformers/models/align/processing_align.py +++ b/src/transformers/models/align/processing_align.py @@ -31,7 +31,7 @@ class AlignProcessor(ProcessorMixin): Args: image_processor ([`EfficientNetImageProcessor`]): The image processor is a required input. - tokenizer ([`BERTTokenizer`, `BertTokenizerFast`]): + tokenizer ([`BertTokenizer`, `BertTokenizerFast`]): The tokenizer is a required input. """ From 0d08783b96591d3077bbe73860dd9be50f2898c4 Mon Sep 17 00:00:00 2001 From: Nipun Jindal Date: Thu, 9 Mar 2023 17:47:44 +0530 Subject: [PATCH 037/277] [21737][T5]: Fix gradient checkpoint bug (#22036) * [21737][T5]: Fix gradient checkpoint bug * [21737][T5]: Fix gradient checkpoint bug * [21737][T5]: Fix gradient checkpoint bug * Update src/transformers/models/mt5/modeling_mt5.py * Update src/transformers/models/t5/modeling_t5.py --------- Co-authored-by: njindal Co-authored-by: Joao Gante --- src/transformers/models/mt5/modeling_mt5.py | 12 +++++++----- src/transformers/models/t5/modeling_t5.py | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/transformers/models/mt5/modeling_mt5.py b/src/transformers/models/mt5/modeling_mt5.py index 951a68cb7678..e2235fc69055 100644 --- a/src/transformers/models/mt5/modeling_mt5.py +++ b/src/transformers/models/mt5/modeling_mt5.py @@ -978,6 +978,13 @@ def forward( else: encoder_extended_attention_mask = None + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + # Prepare head mask if needed head_mask = self.get_head_mask(head_mask, self.config.num_layers) cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers) @@ -1015,11 +1022,6 @@ def forward( all_hidden_states = all_hidden_states + (hidden_states,) if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py index 976899863119..19cb83dac352 100644 --- a/src/transformers/models/t5/modeling_t5.py +++ b/src/transformers/models/t5/modeling_t5.py @@ -1007,6 +1007,13 @@ def forward( else: encoder_extended_attention_mask = None + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + # Prepare head mask if needed head_mask = self.get_head_mask(head_mask, self.config.num_layers) cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers) @@ -1044,11 +1051,6 @@ def forward( all_hidden_states = all_hidden_states + (hidden_states,) if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 67bd8ee604866085e0c7abc24033d2d8458ebd20 Mon Sep 17 00:00:00 2001 From: Shaun VanWeelden Date: Thu, 9 Mar 2023 08:43:49 -0400 Subject: [PATCH 038/277] Docs Improvement - In ZSH, not using ' ' around pip install fails, fix it (#22045) In ZSH, not using ' ' around pip install fails Running ``` pip install transformers[torch] ``` in the default ZSH terminal will fail with the error `zsh: no matches found: transformers[torch]` The solution is to wrap the installation path in ' ' like ``` pip install 'transformers[torch]' ``` Relevant StackOverflow: https://stackoverflow.com/questions/30539798/zsh-no-matches-found-requestssecurity --- docs/source/en/installation.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/en/installation.mdx b/docs/source/en/installation.mdx index 4ff4e04436c7..ba41ed6ea398 100644 --- a/docs/source/en/installation.mdx +++ b/docs/source/en/installation.mdx @@ -54,19 +54,19 @@ pip install transformers For CPU-support only, you can conveniently install 🤗 Transformers and a deep learning library in one line. For example, install 🤗 Transformers and PyTorch with: ```bash -pip install transformers[torch] +pip install 'transformers[torch]' ``` 🤗 Transformers and TensorFlow 2.0: ```bash -pip install transformers[tf-cpu] +pip install 'transformers[tf-cpu]' ``` 🤗 Transformers and Flax: ```bash -pip install transformers[flax] +pip install 'transformers[flax]' ``` Finally, check if 🤗 Transformers has been properly installed by running the following command. It will download a pretrained model: @@ -237,4 +237,4 @@ Once your file is downloaded and locally cached, specify it's local path to load See the [How to download files from the Hub](https://huggingface.co/docs/hub/how-to-downstream) section for more details on downloading files stored on the Hub. - \ No newline at end of file + From c2420fddb8c6c312943f3391c68c7d0152ce8d97 Mon Sep 17 00:00:00 2001 From: Shaun VanWeelden Date: Thu, 9 Mar 2023 08:44:58 -0400 Subject: [PATCH 039/277] Can't install tf2 on M1 Chip by default (#22046) --- docs/source/en/installation.mdx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/en/installation.mdx b/docs/source/en/installation.mdx index ba41ed6ea398..2c59d9146ba4 100644 --- a/docs/source/en/installation.mdx +++ b/docs/source/en/installation.mdx @@ -63,6 +63,18 @@ pip install 'transformers[torch]' pip install 'transformers[tf-cpu]' ``` + + +M1 / ARM Users + +You will need to install the following before installing TensorFLow 2.0 +``` +brew install cmake +brew install pkg-config +``` + + + 🤗 Transformers and Flax: ```bash From da98339c6d0164c3e6c2f740d2e23407011c4d7b Mon Sep 17 00:00:00 2001 From: Lucain Date: Thu, 9 Mar 2023 15:23:48 +0100 Subject: [PATCH 040/277] Remove set_access_token usage + fail tests if FutureWarning (#22051) * Remove set_access_token usage + fail tests if FutureWarning * do not fail on FutureWarning in CI --------- Co-authored-by: testbot --- tests/generation/test_configuration_utils.py | 3 +-- tests/models/auto/test_processor_auto.py | 3 +-- tests/pipelines/test_pipelines_common.py | 3 +-- tests/test_configuration_common.py | 3 +-- tests/test_feature_extraction_common.py | 3 +-- tests/test_image_processing_common.py | 3 +-- tests/test_modeling_common.py | 3 +-- tests/test_modeling_flax_common.py | 3 +-- tests/test_modeling_tf_common.py | 3 +-- tests/test_tokenization_common.py | 3 +-- tests/trainer/test_trainer.py | 3 +-- 11 files changed, 11 insertions(+), 22 deletions(-) diff --git a/tests/generation/test_configuration_utils.py b/tests/generation/test_configuration_utils.py index 8add735a0bf0..a12b35968283 100644 --- a/tests/generation/test_configuration_utils.py +++ b/tests/generation/test_configuration_utils.py @@ -17,7 +17,7 @@ import tempfile import unittest -from huggingface_hub import HfFolder, delete_repo, set_access_token +from huggingface_hub import HfFolder, delete_repo from parameterized import parameterized from requests.exceptions import HTTPError @@ -99,7 +99,6 @@ class ConfigPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/models/auto/test_processor_auto.py b/tests/models/auto/test_processor_auto.py index a880bc0a084a..e0bb4946f702 100644 --- a/tests/models/auto/test_processor_auto.py +++ b/tests/models/auto/test_processor_auto.py @@ -21,7 +21,7 @@ from pathlib import Path from shutil import copyfile -from huggingface_hub import HfFolder, Repository, create_repo, delete_repo, set_access_token +from huggingface_hub import HfFolder, Repository, create_repo, delete_repo from requests.exceptions import HTTPError from transformers import ( @@ -219,7 +219,6 @@ class ProcessorPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 68207b9ca189..f43f439ac279 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -21,7 +21,7 @@ import datasets import numpy as np -from huggingface_hub import HfFolder, Repository, create_repo, delete_repo, set_access_token +from huggingface_hub import HfFolder, Repository, create_repo, delete_repo from requests.exceptions import HTTPError from transformers import ( @@ -759,7 +759,6 @@ class DynamicPipelineTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/test_configuration_common.py b/tests/test_configuration_common.py index dc9927f19385..d2172491f982 100644 --- a/tests/test_configuration_common.py +++ b/tests/test_configuration_common.py @@ -23,7 +23,7 @@ import unittest.mock as mock from pathlib import Path -from huggingface_hub import HfFolder, delete_repo, set_access_token +from huggingface_hub import HfFolder, delete_repo from requests.exceptions import HTTPError from transformers import AutoConfig, BertConfig, GPT2Config, is_torch_available @@ -222,7 +222,6 @@ class ConfigPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/test_feature_extraction_common.py b/tests/test_feature_extraction_common.py index ee8dfefb8406..f26709e2d9a6 100644 --- a/tests/test_feature_extraction_common.py +++ b/tests/test_feature_extraction_common.py @@ -22,7 +22,7 @@ import unittest.mock as mock from pathlib import Path -from huggingface_hub import HfFolder, delete_repo, set_access_token +from huggingface_hub import HfFolder, delete_repo from requests.exceptions import HTTPError from transformers import AutoFeatureExtractor, Wav2Vec2FeatureExtractor @@ -100,7 +100,6 @@ class FeatureExtractorPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/test_image_processing_common.py b/tests/test_image_processing_common.py index 32be6e0e6330..12cdef7a47c7 100644 --- a/tests/test_image_processing_common.py +++ b/tests/test_image_processing_common.py @@ -22,7 +22,7 @@ import unittest.mock as mock from pathlib import Path -from huggingface_hub import HfFolder, delete_repo, set_access_token +from huggingface_hub import HfFolder, delete_repo from requests.exceptions import HTTPError from transformers import AutoImageProcessor, ViTImageProcessor @@ -232,7 +232,6 @@ class ImageProcessorPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 5cb04edbb192..cb06400e9a77 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -30,7 +30,7 @@ from typing import Dict, List, Tuple import numpy as np -from huggingface_hub import HfFolder, delete_repo, set_access_token +from huggingface_hub import HfFolder, delete_repo from huggingface_hub.file_download import http_get from pytest import mark from requests.exceptions import HTTPError @@ -3429,7 +3429,6 @@ class ModelPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/test_modeling_flax_common.py b/tests/test_modeling_flax_common.py index 76e4d6c88154..5578b7893608 100644 --- a/tests/test_modeling_flax_common.py +++ b/tests/test_modeling_flax_common.py @@ -21,7 +21,7 @@ from typing import List, Tuple import numpy as np -from huggingface_hub import HfFolder, delete_repo, set_access_token +from huggingface_hub import HfFolder, delete_repo from requests.exceptions import HTTPError import transformers @@ -1173,7 +1173,6 @@ class FlaxModelPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 42db9aea2698..26e567530ec2 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -28,7 +28,7 @@ from typing import List, Tuple, get_type_hints from datasets import Dataset -from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token +from huggingface_hub import HfFolder, Repository, delete_repo from huggingface_hub.file_download import http_get from requests.exceptions import HTTPError @@ -2409,7 +2409,6 @@ class TFModelPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 965a059c7719..a8bdb7372b8f 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union -from huggingface_hub import HfFolder, delete_repo, set_access_token +from huggingface_hub import HfFolder, delete_repo from huggingface_hub.file_download import http_get from parameterized import parameterized from requests.exceptions import HTTPError @@ -3971,7 +3971,6 @@ class TokenizerPushToHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 2ff81e5fe7bf..274ee76e53c8 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -29,7 +29,7 @@ from unittest.mock import Mock, patch import numpy as np -from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token +from huggingface_hub import HfFolder, Repository, delete_repo from parameterized import parameterized from requests.exceptions import HTTPError @@ -2005,7 +2005,6 @@ class TrainerIntegrationWithHubTester(unittest.TestCase): @classmethod def setUpClass(cls): cls._token = TOKEN - set_access_token(TOKEN) HfFolder.save_token(TOKEN) @classmethod From 6fd5f2fd5ee7ccb2993327dd51c12b3e26ccd141 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 9 Mar 2023 15:39:05 +0100 Subject: [PATCH 041/277] Show the number of `huggingface_hub` warnings in CI report (#22054) * show hfh warnings --------- Co-authored-by: ydshieh --- utils/notification_service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/notification_service.py b/utils/notification_service.py index 25933a0ab218..0aefd5844d32 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -215,11 +215,15 @@ def warnings(self) -> Dict: # Use the actual job link job_link = f"{github_actions_job_links['Extract warnings in CI artifacts']}" + huggingface_hub_warnings = [x for x in self.selected_warnings if "huggingface_hub" in x] + text = f"There are {len(self.selected_warnings)} warnings being selected." + text += f"\n{len(huggingface_hub_warnings)} of them are from `huggingface_hub`." + return { "type": "section", "text": { "type": "plain_text", - "text": f"There were {len(self.selected_warnings)} warnings being selected.", + "text": text, "emoji": True, }, "accessory": { From d95d2d9a3cbfa34a5bc45239cc9adf80a36ea51a Mon Sep 17 00:00:00 2001 From: anruijian <115125339+anruijian@users.noreply.github.com> Date: Thu, 9 Mar 2023 09:44:17 -0500 Subject: [PATCH 042/277] Return analysis for hyperparameter_search with Ray backend (#22040) * return analysis for hyperparameter_search with ray backend * Revert "return analysis for hyperparameter_search with ray backend" This reverts commit cd5179070930e03020d96d98eb51dec3eb21ef75. * add run_summary attribute to BestRun and return analysis for ray backend * fix typo * add doc for run_summary for ray backend --- src/transformers/integrations.py | 2 +- src/transformers/trainer.py | 3 ++- src/transformers/trainer_utils.py | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/transformers/integrations.py b/src/transformers/integrations.py index a2effeac6361..14857f830831 100644 --- a/src/transformers/integrations.py +++ b/src/transformers/integrations.py @@ -346,7 +346,7 @@ def dynamic_modules_import_trainable(*args, **kwargs): **kwargs, ) best_trial = analysis.get_best_trial(metric="objective", mode=direction[:3], scope=trainer.args.ray_scope) - best_run = BestRun(best_trial.trial_id, best_trial.last_result["objective"], best_trial.config) + best_run = BestRun(best_trial.trial_id, best_trial.last_result["objective"], best_trial.config, analysis) if _tb_writer is not None: trainer.add_callback(_tb_writer) return best_run diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 518e58342afe..344523842343 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -2492,7 +2492,8 @@ def hyperparameter_search( - the documentation of [sigopt](https://app.sigopt.com/docs/endpoints/experiments/create) Returns: - [`trainer_utils.BestRun`]: All the information about the best run. + [`trainer_utils.BestRun`]: All the information about the best run. Experiment summary can be found in + `run_summary` attribute for Ray backend. """ if backend is None: backend = default_hp_search_backend() diff --git a/src/transformers/trainer_utils.py b/src/transformers/trainer_utils.py index 9f273ab1ed6c..a213e4b1f447 100644 --- a/src/transformers/trainer_utils.py +++ b/src/transformers/trainer_utils.py @@ -192,7 +192,7 @@ class HubStrategy(ExplicitEnum): class BestRun(NamedTuple): """ - The best run found by an hyperparameter search (see [`~Trainer.hyperparameter_search`]). + The best run found by a hyperparameter search (see [`~Trainer.hyperparameter_search`]). Parameters: run_id (`str`): @@ -202,11 +202,14 @@ class BestRun(NamedTuple): The objective that was obtained for this run. hyperparameters (`Dict[str, Any]`): The hyperparameters picked to get this run. + run_summary (`Optional[Any]`): + A summary of tuning experiments. `ray.tune.ExperimentAnalysis` object for Ray backend. """ run_id: str objective: float hyperparameters: Dict[str, Any] + run_summary: Optional[Any] = None def default_compute_objective(metrics: Dict[str, float]) -> float: From a7dacfb7a4cd134f6e0b92cca80873277718ae58 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 9 Mar 2023 15:36:29 +0000 Subject: [PATCH 043/277] pt-to-tf model architecture override (#22055) * Add an argument to pt-to-tf to allow overriding the model class * make fixup * Minor fix to error message * Remove unused extra conversion from the script --- src/transformers/commands/pt_to_tf.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/transformers/commands/pt_to_tf.py b/src/transformers/commands/pt_to_tf.py index 669f7a98003b..e025db5d1344 100644 --- a/src/transformers/commands/pt_to_tf.py +++ b/src/transformers/commands/pt_to_tf.py @@ -68,6 +68,7 @@ def convert_command_factory(args: Namespace): args.no_pr, args.push, args.extra_commit_description, + args.override_model_class, ) @@ -126,6 +127,13 @@ def register_subcommand(parser: ArgumentParser): default="", help="Optional additional commit description to use when opening a PR (e.g. to tag the owner).", ) + train_parser.add_argument( + "--override-model-class", + type=str, + default=None, + help="If you think you know better than the auto-detector, you can specify the model class here. " + "Can be either an AutoModel class or a specific model class like BertForSequenceClassification.", + ) train_parser.set_defaults(func=convert_command_factory) @staticmethod @@ -175,6 +183,7 @@ def __init__( no_pr: bool, push: bool, extra_commit_description: str, + override_model_class: str, *args, ): self._logger = logging.get_logger("transformers-cli/pt_to_tf") @@ -185,6 +194,7 @@ def __init__( self._no_pr = no_pr self._push = push self._extra_commit_description = extra_commit_description + self._override_model_class = override_model_class def get_inputs(self, pt_model, config): """ @@ -269,7 +279,20 @@ def run(self): # Load config and get the appropriate architecture -- the latter is needed to convert the head's weights config = AutoConfig.from_pretrained(self._local_dir) architectures = config.architectures - if architectures is None: # No architecture defined -- use auto classes + if self._override_model_class is not None: + if self._override_model_class.startswith("TF"): + architectures = [self._override_model_class[2:]] + else: + architectures = [self._override_model_class] + try: + pt_class = getattr(import_module("transformers"), architectures[0]) + except AttributeError: + raise ValueError(f"Model class {self._override_model_class} not found in transformers.") + try: + tf_class = getattr(import_module("transformers"), "TF" + architectures[0]) + except AttributeError: + raise ValueError(f"TF model class TF{self._override_model_class} not found in transformers.") + elif architectures is None: # No architecture defined -- use auto classes pt_class = getattr(import_module("transformers"), "AutoModel") tf_class = getattr(import_module("transformers"), "TFAutoModel") self._logger.warning("No detected architecture, using AutoModel/TFAutoModel") @@ -287,7 +310,6 @@ def run(self): pt_model = pt_class.from_pretrained(self._local_dir) pt_model.eval() - tf_from_pt_model = tf_class.from_pretrained(self._local_dir, from_pt=True) pt_input, tf_input = self.get_inputs(pt_model, config) with torch.no_grad(): From c8437eafa991f25ec57bf0f5d95f761e1c563697 Mon Sep 17 00:00:00 2001 From: Kamal Raj Kanakarajan Date: Thu, 9 Mar 2023 21:39:46 +0530 Subject: [PATCH 044/277] rm $ symbol from code block from contributing.md (#22057) rm $ symbol from code block Removed the $ symbol from the code block to make copy-pasting easier. --- CONTRIBUTING.md | 56 ++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 122a884c58a9..0bf7e59df226 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,15 +139,15 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai 2. Clone your fork to your local disk, and add the base repository as a remote: ```bash - $ git clone git@github.com:/transformers.git - $ cd transformers - $ git remote add upstream https://github.com/huggingface/transformers.git + git clone git@github.com:/transformers.git + cd transformers + git remote add upstream https://github.com/huggingface/transformers.git ``` 3. Create a new branch to hold your development changes: ```bash - $ git checkout -b a-descriptive-name-for-my-changes + git checkout -b a-descriptive-name-for-my-changes ``` 🚨 **Do not** work on the `main` branch! @@ -155,7 +155,7 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai 4. Set up a development environment by running the following command in a virtual environment: ```bash - $ pip install -e ".[dev]" + pip install -e ".[dev]" ``` If 🤗 Transformers was already installed in the virtual environment, remove @@ -176,7 +176,7 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai passes. Run the tests impacted by your changes like this: ```bash - $ pytest tests/.py + pytest tests/.py ``` For more information about tests, check out the @@ -187,7 +187,7 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai that can't be automated in one go with: ```bash - $ make fixup + make fixup ``` This target is also optimized to only work with files modified by the PR you're working on. @@ -196,21 +196,21 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai style corrections: ```bash - $ make style + make style ``` 🤗 Transformers also uses `ruff` and a few custom scripts to check for coding mistakes. Quality controls are run by the CI, but you can run the same checks with: ```bash - $ make quality + make quality ``` Finally, we have a lot of scripts to make sure we didn't forget to update some files when adding a new model. You can run these scripts with: ```bash - $ make repo-consistency + make repo-consistency ``` To learn more about those checks and how to fix any issues with them, check out the @@ -220,13 +220,13 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai make sure you install the documentation builder: ```bash - $ pip install ".[docs]" + pip install ".[docs]" ``` Run the following command from the root of the repository: ```bash - $ doc-builder build transformers docs/source/en --build_dir ~/tmp/test-build + doc-builder build transformers docs/source/en --build_dir ~/tmp/test-build ``` This will build the documentation in the `~/tmp/test-build` folder where you can inspect the generated @@ -236,8 +236,8 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai record your changes locally with `git commit`: ```bash - $ git add modified_file.py - $ git commit + git add modified_file.py + git commit ``` Please remember to write [good commit @@ -247,14 +247,14 @@ You'll need **[Python 3.7]((https://github.com/huggingface/transformers/blob/mai repository, rebase your branch on `upstream/branch` *before* you open a pull request or if requested by a maintainer: ```bash - $ git fetch upstream - $ git rebase upstream/main + git fetch upstream + git rebase upstream/main ``` Push your changes to your branch: ```bash - $ git push -u origin a-descriptive-name-for-my-changes + git push -u origin a-descriptive-name-for-my-changes ``` If you've already opened a pull request, you'll need to force push with the `--force` flag. Otherwise, if the pull request hasn't been opened yet, you can just push your changes normally. @@ -307,14 +307,14 @@ We like `pytest` and `pytest-xdist` because it's faster. From the root of the repository, specify a *path to a subfolder or a test file* to run the test. ```bash -$ python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model +python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model ``` Similarly, for the `examples` directory, specify a *path to a subfolder or test file* to run the test. For example, the following command tests the text classification subfolder in the PyTorch `examples` directory: ```bash -$ pip install -r examples/xxx/requirements.txt # only needed the first time -$ python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification +pip install -r examples/xxx/requirements.txt # only needed the first time +python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification ``` In fact, this is actually how our `make test` and `make test-examples` commands are implemented (not including the `pip install`)! @@ -333,8 +333,8 @@ Remember to specify a *path to a subfolder or a test file* to run the test. Othe ```bash -$ RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model -$ RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification +RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./tests/models/my_new_model +RUN_SLOW=yes python -m pytest -n auto --dist=loadfile -s -v ./examples/pytorch/text-classification ``` Like the slow tests, there are other environment variables available which not enabled by default during testing: @@ -351,8 +351,8 @@ This means `unittest` is fully supported. Here's how to run tests with `unittest`: ```bash -$ python -m unittest discover -s tests -t . -v -$ python -m unittest discover -s examples -t examples -v +python -m unittest discover -s tests -t . -v +python -m unittest discover -s examples -t examples -v ``` ### Style guide @@ -386,8 +386,8 @@ When updating the main branch of a forked repository, please follow these steps 2. If a PR is absolutely necessary, use the following steps after checking out your branch: ```bash -$ git checkout -b your-branch-for-syncing -$ git pull --squash --no-commit upstream main -$ git commit -m '' -$ git push --set-upstream origin your-branch-for-syncing +git checkout -b your-branch-for-syncing +git pull --squash --no-commit upstream main +git commit -m '' +git push --set-upstream origin your-branch-for-syncing ``` From 09e9344c491afc4148b2541d7a2f5de4b249ab04 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 9 Mar 2023 08:12:57 -0800 Subject: [PATCH 045/277] [deepspeed] offload + non-cpuadam optimizer exception (#22043) * [deepspeed] offload + non-cpuadam optimizer exception * flip * revert min version --- tests/deepspeed/test_deepspeed.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/deepspeed/test_deepspeed.py b/tests/deepspeed/test_deepspeed.py index 9b203dfd7b95..ba9c269cd107 100644 --- a/tests/deepspeed/test_deepspeed.py +++ b/tests/deepspeed/test_deepspeed.py @@ -426,6 +426,7 @@ def test_hf_optimizer_with_offload(self, stage, dtype): del ds_config_dict["optimizer"] # force default HF Trainer optimizer # force cpu offload ds_config_dict["zero_optimization"]["offload_optimizer"]["device"] = "cpu" + ds_config_dict["zero_force_ds_cpu_optimizer"] = False # offload is not efficient w/o CPUAdam with mockenv_context(**self.dist_env_1_gpu): kwargs = {"local_rank": 0, "deepspeed": ds_config_dict} kwargs[dtype] = True @@ -776,6 +777,7 @@ def test_load_best_model(self, stage, dtype): ds_config_dict = self.get_config_dict(stage) del ds_config_dict["optimizer"] # will use HF Trainer optimizer del ds_config_dict["scheduler"] # will use HF Trainer scheduler + ds_config_dict["zero_force_ds_cpu_optimizer"] = False # offload is not efficient w/o CPUAdam # must use this setting to get the reload path exercised ds_config_dict["zero_optimization"]["stage3_gather_16bit_weights_on_model_save"] = True From e39722e37627b0f4013fb7f9af137c8e7132e8b7 Mon Sep 17 00:00:00 2001 From: Jiali Mei Date: Thu, 9 Mar 2023 18:35:43 +0100 Subject: [PATCH 046/277] Edit the docstring of `image_processing_donut` to match code (#22033) * Edit the docstring of `image_processing_donut` to match code * improve style * more style improvement after installing quality --- .../models/donut/image_processing_donut.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/donut/image_processing_donut.py b/src/transformers/models/donut/image_processing_donut.py index 325a2bb9b602..c94927472915 100644 --- a/src/transformers/models/donut/image_processing_donut.py +++ b/src/transformers/models/donut/image_processing_donut.py @@ -63,12 +63,14 @@ class DonutImageProcessor(BaseImageProcessor): method. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): Resampling filter to use if resizing the image. Can be overridden by `resample` in the `preprocess` method. - do_center_crop (`bool`, *optional*, defaults to `True`): - Whether to center crop the image to the specified `crop_size`. Can be overridden by `do_center_crop` in the - `preprocess` method. - crop_size (`Dict[str, int]` *optional*, defaults to 224): - Size of the output image after applying `center_crop`. Can be overridden by `crop_size` in the `preprocess` - method. + do_thumbnail (`bool`, *optional*, defaults to `True`): + Whether to resize the image using thumbnail method. + do_align_long_axis (`bool`, *optional*, defaults to `False`): + Whether to align the long axis of the image with the long axis of `size` by rotating by 90 degrees. + do_pad (`bool`, *optional*, defaults to `True`): + Whether to pad the image. If `random_padding` is set to `True` in `preprocess`, each image is padded with a + random amont of padding on each size, up to the largest image size in the batch. Otherwise, all images are + padded to the largest image size in the batch. do_rescale (`bool`, *optional*, defaults to `True`): Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by `do_rescale` in the `preprocess` method. @@ -82,9 +84,6 @@ class DonutImageProcessor(BaseImageProcessor): channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): Image standard deviation. - do_convert_rgb (`bool`, *optional*, defaults to `True`): - Standard deviation to use if normalizing the image. This is a float or list of floats the length of the - number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. """ model_input_names = ["pixel_values"] From bf97e229b5e090b8127387dcb7e39f263b22accc Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 9 Mar 2023 19:09:23 +0100 Subject: [PATCH 047/277] Skip 3 tests for `WhisperEncoderModelTest` (#22060) * skip 3 tests --------- Co-authored-by: ydshieh --- tests/models/whisper/test_modeling_whisper.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/models/whisper/test_modeling_whisper.py b/tests/models/whisper/test_modeling_whisper.py index 4ceb30cfeaa2..9d99c2d4d5e5 100644 --- a/tests/models/whisper/test_modeling_whisper.py +++ b/tests/models/whisper/test_modeling_whisper.py @@ -1525,6 +1525,18 @@ def test_forward_signature(self): expected_arg_names = ["input_features", "head_mask", "encoder_outputs"] self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names) + @unittest.skip(reason="Some undefined behavior encountered with tiny versions of this model. Skip for now.") + def test_cpu_offload(self): + pass + + @unittest.skip(reason="Some undefined behavior encountered with tiny versions of this model. Skip for now.") + def test_disk_offload(self): + pass + + @unittest.skip(reason="Some undefined behavior encountered with tiny versions of this model. Skip for now.") + def test_model_parallelism(self): + pass + # input embeds is meaningless for an encoder-only acoustic model def test_inputs_embeds(self): pass From 4405fa433a2339eab00e9aa05995f9931508ae82 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Thu, 9 Mar 2023 13:13:23 -0500 Subject: [PATCH 048/277] Add setters by type of args to TrainingArguments (#21570) * Add setters by type of args to TrainingArguments * Define more setters --- src/transformers/training_args.py | 520 +++++++++++++++++++++++++++++- 1 file changed, 518 insertions(+), 2 deletions(-) diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index de9b36a445c2..67a796e713e6 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -474,8 +474,6 @@ class TrainingArguments: The optimizer to use: adamw_hf, adamw_torch, adamw_apex_fused, adamw_anyprecision or adafactor. optim_args (`str`, *optional*): Optional arguments that are supplied to AnyPrecisionAdamW. - adafactor (`bool`, *optional*, defaults to `False`): - This argument is deprecated. Use `--optim adafactor` instead. group_by_length (`bool`, *optional*, defaults to `False`): Whether or not to group together samples of roughly the same length in the training dataset (to minimize padding applied and be more efficient). Only useful if applying dynamic padding. @@ -1905,6 +1903,524 @@ def to_sanitized_dict(self) -> Dict[str, Any]: return {k: v if type(v) in valid_types else str(v) for k, v in d.items()} + # The following methods are there to simplify the instantiation of `TrainingArguments` + def set_training( + self, + learning_rate: float = 5e-5, + batch_size: int = 8, + weight_decay: float = 0, + num_epochs: float = 3, + max_steps: int = -1, + gradient_accumulation_steps: int = 1, + seed: int = 42, + gradient_checkpointing: bool = False, + ): + """ + A method that regroups all basic arguments linked to the training. + + + + Calling this method will automatically set `self.do_train` to `True`. + + + + Args: + learning_rate (`float`, *optional*, defaults to 5e-5): + The initial learning rate for the optimizer. + batch_size (`int` *optional*, defaults to 8): + The batch size per device (GPU/TPU core/CPU...) used for training. + weight_decay (`float`, *optional*, defaults to 0): + The weight decay to apply (if not zero) to all layers except all bias and LayerNorm weights in the + optimizer. + num_train_epochs(`float`, *optional*, defaults to 3.0): + Total number of training epochs to perform (if not an integer, will perform the decimal part percents + of the last epoch before stopping training). + max_steps (`int`, *optional*, defaults to -1): + If set to a positive number, the total number of training steps to perform. Overrides + `num_train_epochs`. In case of using a finite iterable dataset the training may stop before reaching + the set number of steps when all data is exhausted. + gradient_accumulation_steps (`int`, *optional*, defaults to 1): + Number of updates steps to accumulate the gradients for, before performing a backward/update pass. + + + + When using gradient accumulation, one step is counted as one step with backward pass. Therefore, + logging, evaluation, save will be conducted every `gradient_accumulation_steps * xxx_step` training + examples. + + + + seed (`int`, *optional*, defaults to 42): + Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use + the [`~Trainer.model_init`] function to instantiate the model if it has some randomly initialized + parameters. + gradient_checkpointing (`bool`, *optional*, defaults to `False`): + If True, use gradient checkpointing to save memory at the expense of slower backward pass. + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_training(learning_rate=1e-4, batch_size=32) + >>> args.learning_rate + 1e-4 + ``` + """ + self.do_train = True + self.learning_rate = learning_rate + self.per_device_train_batch_size = batch_size + self.weight_decay = weight_decay + self.num_train_epochs = num_epochs + self.max_steps = max_steps + self.gradient_accumulation_steps = gradient_accumulation_steps + self.seed = seed + self.gradient_checkpointing = gradient_checkpointing + return self + + def set_evaluate( + self, + strategy: Union[str, IntervalStrategy] = "no", + steps: int = 500, + batch_size: int = 8, + accumulation_steps: Optional[int] = None, + delay: Optional[float] = None, + loss_only: bool = False, + jit_mode: bool = False, + ): + """ + A method that regroups all arguments linked to the evaluation. + + Args: + strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`): + The evaluation strategy to adopt during training. Possible values are: + + - `"no"`: No evaluation is done during training. + - `"steps"`: Evaluation is done (and logged) every `steps`. + - `"epoch"`: Evaluation is done at the end of each epoch. + + Setting a `strategy` different from `"no"` will set `self.do_eval` to `True`. + steps (`int`, *optional*, defaults to 500): + Number of update steps between two evaluations if `strategy="steps"`. + batch_size (`int` *optional*, defaults to 8): + The batch size per device (GPU/TPU core/CPU...) used for evaluation. + accumulation_steps (`int`, *optional*): + Number of predictions steps to accumulate the output tensors for, before moving the results to the CPU. + If left unset, the whole predictions are accumulated on GPU/TPU before being moved to the CPU (faster + but requires more memory). + delay (`float`, *optional*): + Number of epochs or steps to wait for before the first evaluation can be performed, depending on the + evaluation_strategy. + loss_only (`bool`, *optional*, defaults to `False`): + Ignores all outputs except the loss. + jit_mode (`bool`, *optional*): + Whether or not to use PyTorch jit trace for inference. + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_evaluate(strategy="steps", steps=100) + >>> args.eval_steps + 100 + ``` + """ + self.evaluation_strategy = IntervalStrategy(strategy) + if self.evaluation_strategy == IntervalStrategy.STEPS and steps == 0: + raise ValueError("Setting `strategy` as 'steps' requires a positive value for `steps`.") + self.do_eval = self.evaluation_strategy != IntervalStrategy.NO + self.eval_steps = steps + self.per_device_eval_batch_size = batch_size + self.eval_accumulation_steps = accumulation_steps + self.eval_delay = delay + self.prediction_loss_only = loss_only + self.jit_mode_eval = jit_mode + return self + + def set_testing( + self, + batch_size: int = 8, + loss_only: bool = False, + jit_mode: bool = False, + ): + """ + A method that regroups all basic arguments linked to testing on a held-out dataset. + + + + Calling this method will automatically set `self.do_predict` to `True`. + + + + Args: + batch_size (`int` *optional*, defaults to 8): + The batch size per device (GPU/TPU core/CPU...) used for testing. + loss_only (`bool`, *optional*, defaults to `False`): + Ignores all outputs except the loss. + jit_mode (`bool`, *optional*): + Whether or not to use PyTorch jit trace for inference. + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_testing(batch_size=32) + >>> args.per_device_eval_batch_size + 32 + ``` + """ + self.do_predict = True + self.per_device_eval_batch_size = batch_size + self.prediction_loss_only = loss_only + self.jit_mode_eval = jit_mode + return self + + def set_save( + self, + strategy: Union[str, IntervalStrategy] = "steps", + steps: int = 500, + total_limit: Optional[int] = None, + on_each_node: bool = False, + ): + """ + A method that regroups all arguments linked to the evaluation. + + Args: + strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): + The checkpoint save strategy to adopt during training. Possible values are: + + - `"no"`: No save is done during training. + - `"epoch"`: Save is done at the end of each epoch. + - `"steps"`: Save is done every `save_steps`. + + steps (`int`, *optional*, defaults to 500): + Number of updates steps before two checkpoint saves if `strategy="steps"`. + total_limit (`int`, *optional*): + If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in + `output_dir`. + on_each_node (`bool`, *optional*, defaults to `False`): + When doing multi-node distributed training, whether to save models and checkpoints on each node, or + only on the main one. + + This should not be activated when the different nodes use the same storage as the files will be saved + with the same names for each node. + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_save(strategy="steps", steps=100) + >>> args.save_steps + 100 + ``` + """ + self.save_strategy = IntervalStrategy(strategy) + if self.save_strategy == IntervalStrategy.STEPS and steps == 0: + raise ValueError("Setting `strategy` as 'steps' requires a positive value for `steps`.") + self.save_steps = steps + self.save_total_limit = total_limit + self.save_on_each_node = on_each_node + return self + + def set_logging( + self, + strategy: Union[str, IntervalStrategy] = "steps", + steps: int = 500, + report_to: Union[str, List[str]] = "none", + level: str = "passive", + first_step: bool = False, + nan_inf_filter: bool = False, + on_each_node: bool = False, + replica_level: str = "passive", + ): + """ + A method that regroups all arguments linked to the evaluation. + + Args: + strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): + The logging strategy to adopt during training. Possible values are: + + - `"no"`: No save is done during training. + - `"epoch"`: Save is done at the end of each epoch. + - `"steps"`: Save is done every `save_steps`. + + steps (`int`, *optional*, defaults to 500): + Number of update steps between two logs if `strategy="steps"`. + level (`str`, *optional*, defaults to `"passive"`): + Logger log level to use on the main process. Possible choices are the log levels as strings: `"debug"`, + `"info"`, `"warning"`, `"error"` and `"critical"`, plus a `"passive"` level which doesn't set anything + and lets the application set the level. + report_to (`str` or `List[str]`, *optional*, defaults to `"none"`): + The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`, + `"comet_ml"`, `"mlflow"`, `"neptune"`, `"tensorboard"`,`"clearml"` and `"wandb"`. Use `"all"` to report + to all integrations installed, `"none"` for no integrations. + first_step (`bool`, *optional*, defaults to `False`): + Whether to log and evaluate the first `global_step` or not. + nan_inf_filter (`bool`, *optional*, defaults to `True`): + Whether to filter `nan` and `inf` losses for logging. If set to `True` the loss of every step that is + `nan` or `inf` is filtered and the average loss of the current logging window is taken instead. + + + + `nan_inf_filter` only influences the logging of loss values, it does not change the behavior the + gradient is computed or applied to the model. + + + + on_each_node (`bool`, *optional*, defaults to `True`): + In multinode distributed training, whether to log using `log_level` once per node, or only on the main + node. + replica_level (`str`, *optional*, defaults to `"passive"`): + Logger log level to use on replicas. Same choices as `log_level` + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_logging(strategy="steps", steps=100) + >>> args.logging_steps + 100 + ``` + """ + self.logging_strategy = IntervalStrategy(strategy) + if self.logging_strategy == IntervalStrategy.STEPS and steps == 0: + raise ValueError("Setting `strategy` as 'steps' requires a positive value for `steps`.") + self.logging_steps = steps + self.report_to = report_to + self.log_level = level + self.logging_first_step = first_step + self.logging_nan_inf_filter = nan_inf_filter + self.log_on_each_node = on_each_node + self.log_level_replica = replica_level + return self + + def set_push_to_hub( + self, + model_id: str, + strategy: Union[str, HubStrategy] = "every_save", + token: Optional[str] = None, + private_repo: bool = False, + ): + """ + A method that regroups all arguments linked to synchronizing checkpoints with the Hub. + + + + Calling this method will set `self.push_to_hub` to `True`, which means the `output_dir` will begin a git + directory synced with the repo (determined by `model_id`) and the content will be pushed each time a save is + triggered (depending on`self.save_strategy`). Calling [`~Trainer.save_model`] will also trigger a push. + + + + Args: + model_id (`str`): + The name of the repository to keep in sync with the local *output_dir*. It can be a simple model ID in + which case the model will be pushed in your namespace. Otherwise it should be the whole repository + name, for instance `"user_name/model"`, which allows you to push to an organization you are a member of + with `"organization_name/model"`. + strategy (`str` or [`~trainer_utils.HubStrategy`], *optional*, defaults to `"every_save"`): + Defines the scope of what is pushed to the Hub and when. Possible values are: + + - `"end"`: push the model, its configuration, the tokenizer (if passed along to the [`Trainer`]) and a + draft of a model card when the [`~Trainer.save_model`] method is called. + - `"every_save"`: push the model, its configuration, the tokenizer (if passed along to the [`Trainer`]) + and + a draft of a model card each time there is a model save. The pushes are asynchronous to not block + training, and in case the save are very frequent, a new push is only attempted if the previous one is + finished. A last push is made with the final model at the end of training. + - `"checkpoint"`: like `"every_save"` but the latest checkpoint is also pushed in a subfolder named + last-checkpoint, allowing you to resume training easily with + `trainer.train(resume_from_checkpoint="last-checkpoint")`. + - `"all_checkpoints"`: like `"checkpoint"` but all checkpoints are pushed like they appear in the + output + folder (so you will get one checkpoint folder per folder in your final repository) + + token (`str`, *optional*): + The token to use to push the model to the Hub. Will default to the token in the cache folder obtained + with `huggingface-cli login`. + private_repo (`bool`, *optional*, defaults to `False`): + If True, the Hub repo will be set to private. + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_push_to_hub("me/awesome-model") + >>> args.hub_model_id + 'me/awesome-model' + ``` + """ + self.push_to_hub = True + self.hub_model_id = model_id + self.hub_strategy = HubStrategy(strategy) + self.hub_token = token + self.hub_private_repo = private_repo + return self + + def set_optimizer( + self, + name: Union[str, OptimizerNames] = "adamw_hf", + learning_rate: float = 5e-5, + weight_decay: float = 0, + beta1: float = 0.9, + beta2: float = 0.999, + epsilon: float = 1e-8, + args: Optional[str] = None, + ): + """ + A method that regroups all arguments linked to the optimizer and its hyperparameters. + + Args: + name (`str` or [`training_args.OptimizerNames`], *optional*, defaults to `"adamw_hf"`): + The optimizer to use: `"adamw_hf"`, `"adamw_torch"`, `"adamw_apex_fused"`, `"adamw_anyprecision"` or + `"adafactor"`. + learning_rate (`float`, *optional*, defaults to 5e-5): + The initial learning rate. + weight_decay (`float`, *optional*, defaults to 0): + The weight decay to apply (if not zero) to all layers except all bias and LayerNorm weights. + beta1 (`float`, *optional*, defaults to 0.9): + The beta1 hyperparameter for the adam optimizer or its variants. + beta2 (`float`, *optional*, defaults to 0.999): + The beta2 hyperparameter for the adam optimizer or its variants. + epsilon (`float`, *optional*, defaults to 1e-8): + The epsilon hyperparameter for the adam optimizer or its variants. + args (`str`, *optional*): + Optional arguments that are supplied to AnyPrecisionAdamW (only useful when + `optim="adamw_anyprecision"`). + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_optimizer(name="adamw_torch", beta1=0.8) + >>> args.optim + 'adamw_torch' + ``` + """ + self.optim = OptimizerNames(name) + self.learning_rate = learning_rate + self.weight_decay = weight_decay + self.adam_beta1 = beta1 + self.adam_beta2 = beta2 + self.adam_epsilon = epsilon + self.optim_args = args + return self + + def set_lr_scheduler( + self, + name: Union[str, SchedulerType] = "linear", + num_epochs: float = 3.0, + max_steps: int = -1, + warmup_ratio: float = 0, + warmup_steps: int = 0, + ): + """ + A method that regroups all arguments linked to the learning rate scheduler and its hyperparameters. + + Args: + name (`str` or [`SchedulerType`], *optional*, defaults to `"linear"`): + The scheduler type to use. See the documentation of [`SchedulerType`] for all possible values. + num_epochs(`float`, *optional*, defaults to 3.0): + Total number of training epochs to perform (if not an integer, will perform the decimal part percents + of the last epoch before stopping training). + max_steps (`int`, *optional*, defaults to -1): + If set to a positive number, the total number of training steps to perform. Overrides + `num_train_epochs`. In case of using a finite iterable dataset the training may stop before reaching + the set number of steps when all data is exhausted. + warmup_ratio (`float`, *optional*, defaults to 0.0): + Ratio of total training steps used for a linear warmup from 0 to `learning_rate`. + warmup_steps (`int`, *optional*, defaults to 0): + Number of steps used for a linear warmup from 0 to `learning_rate`. Overrides any effect of + `warmup_ratio`. + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_lr_scheduler(name="cosine", warmup_ratio=0.05) + >>> args.warmup_ratio + 0.05 + ``` + """ + self.lr_scheduler_type = SchedulerType(name) + self.num_train_epochs = num_epochs + self.max_steps = max_steps + self.warmup_ratio = warmup_ratio + self.warmup_steps = warmup_steps + return self + + def set_dataloader( + self, + train_batch_size: int = 8, + eval_batch_size: int = 8, + drop_last: bool = False, + num_workers: int = 0, + pin_memory: bool = True, + auto_find_batch_size: bool = False, + ignore_data_skip: bool = False, + sampler_seed: Optional[int] = None, + ): + """ + A method that regroups all arguments linked to the dataloaders creation. + + Args: + drop_last (`bool`, *optional*, defaults to `False`): + Whether to drop the last incomplete batch (if the length of the dataset is not divisible by the batch + size) or not. + num_workers (`int`, *optional*, defaults to 0): + Number of subprocesses to use for data loading (PyTorch only). 0 means that the data will be loaded in + the main process. + pin_memory (`bool`, *optional*, defaults to `True`): + Whether you want to pin memory in data loaders or not. Will default to `True`. + auto_find_batch_size (`bool`, *optional*, defaults to `False`) + Whether to find a batch size that will fit into memory automatically through exponential decay, + avoiding CUDA Out-of-Memory errors. Requires accelerate to be installed (`pip install accelerate`) + ignore_data_skip (`bool`, *optional*, defaults to `False`): + When resuming training, whether or not to skip the epochs and batches to get the data loading at the + same stage as in the previous training. If set to `True`, the training will begin faster (as that + skipping step can take a long time) but will not yield the same results as the interrupted training + would have. + sampler_seed (`int`, *optional*): + Random seed to be used with data samplers. If not set, random generators for data sampling will use the + same seed as `self.seed`. This can be used to ensure reproducibility of data sampling, independent of + the model seed. + + Example: + + ```py + >>> from transformers import TrainingArguments + + >>> args = TrainingArguments("working_dir") + >>> args = args.set_dataloader(train_batch_size=16, eval_batch_size=64) + >>> args.per_device_train_batch_size + 16 + ``` + """ + self.per_device_train_batch_size = train_batch_size + self.per_device_eval_batch_size = eval_batch_size + self.dataloader_drop_last = drop_last + self.dataloader_num_workers = num_workers + self.dataloader_pin_memory = pin_memory + self.auto_find_batch_size = auto_find_batch_size + self.ignore_data_skip = ignore_data_skip + self.data_seed = sampler_seed + return self + class ParallelMode(Enum): NOT_PARALLEL = "not_parallel" From a4e1c7b7746a3ea4248be30ed715fc5feb180510 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 9 Mar 2023 19:53:54 +0100 Subject: [PATCH 049/277] Update tiny model creation script (#22058) Update the script Co-authored-by: ydshieh --- utils/create_dummy_models.py | 245 ++++++++++++++++++++++++++--------- 1 file changed, 185 insertions(+), 60 deletions(-) diff --git a/utils/create_dummy_models.py b/utils/create_dummy_models.py index 60e055d67063..176bd07dfaaa 100644 --- a/utils/create_dummy_models.py +++ b/utils/create_dummy_models.py @@ -23,6 +23,7 @@ import shutil import sys import tempfile +import traceback from pathlib import Path from check_config_docstrings import get_checkpoint_from_config_class @@ -71,6 +72,67 @@ TARGET_VOCAB_SIZE = 1024 +# This list contains the model architectures for which a tiny version could not be created. +# Avoid to add new architectures here - unless we have verified carefully that it's (almost) impossible to create them. +# One such case is: no model tester class is implemented for a model type (like `MT5`) because its architecture is +# identical to another one (`MT5` is based on `T5`), but trained on different datasets or with different techniques. +UNCONVERTIBLE_MODEL_ARCHITECTURES = { + "BertGenerationEncoder", + "BertGenerationDecoder", + "CamembertForSequenceClassification", + "CamembertForMultipleChoice", + "CamembertForMaskedLM", + "CamembertForCausalLM", + "CamembertForTokenClassification", + "CamembertForQuestionAnswering", + "CamembertModel", + "TFCamembertForMultipleChoice", + "TFCamembertForTokenClassification", + "TFCamembertForQuestionAnswering", + "TFCamembertForSequenceClassification", + "TFCamembertForMaskedLM", + "TFCamembertModel", + "TFCamembertForCausalLM", + "DecisionTransformerModel", + "JukeboxModel", + "MarianForCausalLM", + "MT5Model", + "MT5ForConditionalGeneration", + "TFMT5ForConditionalGeneration", + "TFMT5Model", + "QDQBertForSequenceClassification", + "QDQBertForMaskedLM", + "QDQBertModel", + "QDQBertForTokenClassification", + "QDQBertLMHeadModel", + "QDQBertForMultipleChoice", + "QDQBertForQuestionAnswering", + "QDQBertForNextSentencePrediction", + "ReformerModelWithLMHead", + "RetriBertModel", + "Speech2Text2ForCausalLM", + "TimeSeriesTransformerModel", + "TrajectoryTransformerModel", + "TrOCRForCausalLM", + "XLMProphetNetForConditionalGeneration", + "XLMProphetNetForCausalLM", + "XLMProphetNetModel", + "XLMRobertaModel", + "XLMRobertaForTokenClassification", + "XLMRobertaForMultipleChoice", + "XLMRobertaForMaskedLM", + "XLMRobertaForCausalLM", + "XLMRobertaForSequenceClassification", + "XLMRobertaForQuestionAnswering", + "TFXLMRobertaForSequenceClassification", + "TFXLMRobertaForMaskedLM", + "TFXLMRobertaForQuestionAnswering", + "TFXLMRobertaModel", + "TFXLMRobertaForMultipleChoice", + "TFXLMRobertaForTokenClassification", +} + + def get_processor_types_from_config_class(config_class, allowed_mappings=None): """Return a tuple of processors for `config_class`. @@ -131,7 +193,7 @@ def get_architectures_from_config_class(config_class, arch_mappings): models = mapping[config_class] models = tuple(models) if isinstance(models, collections.abc.Sequence) else (models,) for model in models: - if model.__name__ not in unexportable_model_architectures: + if model.__name__ not in UNCONVERTIBLE_MODEL_ARCHITECTURES: architectures.add(model) architectures = tuple(architectures) @@ -186,8 +248,7 @@ def build_processor(config_class, processor_class, allow_no_checkpoint=False): try: processor = processor_class.from_pretrained(checkpoint) except Exception as e: - logger.error(e) - pass + logger.error(f"{e.__class__.__name__}: {e}") # Try to get a new processor class from checkpoint. This is helpful for a checkpoint without necessary file to load # processor while `processor_class` is an Auto class. For example, `sew` has `Wav2Vec2Processor` in @@ -203,7 +264,7 @@ def build_processor(config_class, processor_class, allow_no_checkpoint=False): try: config = AutoConfig.from_pretrained(checkpoint) except Exception as e: - logger.error(e) + logger.error(f"{e.__class__.__name__}: {e}") config = None if config is not None: if not isinstance(config, config_class): @@ -263,8 +324,7 @@ def build_processor(config_class, processor_class, allow_no_checkpoint=False): try: processor = processor_class(**{k: v[0] for k, v in attrs.items()}) except Exception as e: - logger.error(e) - pass + logger.error(f"{e.__class__.__name__}: {e}") else: # `checkpoint` might lack some file(s) to load a processor. For example, `facebook/hubert-base-ls960` # has no tokenizer file to load `Wav2Vec2CTCTokenizer`. In this case, we try to build a processor @@ -282,8 +342,7 @@ def build_processor(config_class, processor_class, allow_no_checkpoint=False): try: processor = processor_class() except Exception as e: - logger.error(e) - pass + logger.error(f"{e.__class__.__name__}: {e}") # validation if processor is not None: @@ -322,12 +381,12 @@ def get_tiny_config(config_class, **model_tester_kwargs): module = importlib.import_module(f".models.{module_name}.test_modeling_{modeling_name}", package="tests") camel_case_model_name = config_class.__name__.split("Config")[0] model_tester_class = getattr(module, f"{camel_case_model_name}ModelTester", None) - except ModuleNotFoundError as e: - error = f"Tiny config not created for {model_type} - cannot find the testing module from the model name" - raise ValueError(f"{error}: {e}") + except ModuleNotFoundError: + error = f"Tiny config not created for {model_type} - cannot find the testing module from the model name." + raise ValueError(error) if model_tester_class is None: - error = f"Tiny config not created for {model_type} - no model tester is found in the testing module" + error = f"Tiny config not created for {model_type} - no model tester is found in the testing module." raise ValueError(error) # `parent` is an instance of `unittest.TestCase`, but we don't need it here. @@ -434,9 +493,12 @@ def convert_processors(processors, tiny_config, output_folder, result): # be retrained if fast_tokenizer.vocab_size > TARGET_VOCAB_SIZE: fast_tokenizer = convert_tokenizer(tokenizer) - except Exception as e: + except Exception: result["warnings"].append( - f"Failed to convert the fast tokenizer for {fast_tokenizer.__class__.__name__}: {e}" + ( + f"Failed to convert the fast tokenizer for {fast_tokenizer.__class__.__name__}.", + traceback.format_exc(), + ) ) continue elif slow_tokenizer is None: @@ -446,9 +508,12 @@ def convert_processors(processors, tiny_config, output_folder, result): if fast_tokenizer: try: fast_tokenizer.save_pretrained(output_folder) - except Exception as e: + except Exception: result["warnings"].append( - f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}: {e}" + ( + f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.", + traceback.format_exc(), + ) ) fast_tokenizer = None @@ -456,9 +521,12 @@ def convert_processors(processors, tiny_config, output_folder, result): if fast_tokenizer: try: slow_tokenizer = AutoTokenizer.from_pretrained(output_folder, use_fast=False) - except Exception as e: + except Exception: result["warnings"].append( - f"Failed to load the slow tokenizer saved from {fast_tokenizer.__class__.__name__}: {e}" + ( + f"Failed to load the slow tokenizer saved from {fast_tokenizer.__class__.__name__}.", + traceback.format_exc(), + ) ) # Let's just keep the fast version slow_tokenizer = None @@ -467,17 +535,25 @@ def convert_processors(processors, tiny_config, output_folder, result): if not fast_tokenizer and slow_tokenizer: try: slow_tokenizer.save_pretrained(output_folder) - except Exception as e: + except Exception: result["warnings"].append( - f"Failed to save the slow tokenizer for {slow_tokenizer.__class__.__name__}: {e}" + ( + f"Failed to save the slow tokenizer for {slow_tokenizer.__class__.__name__}.", + traceback.format_exc(), + ) ) slow_tokenizer = None # update feature extractors using the tiny config try: feature_extractors = [convert_feature_extractor(p, tiny_config) for p in feature_extractors] - except Exception as e: - result["warnings"].append(f"Failed to convert feature extractors: {e}") + except Exception: + result["warnings"].append( + ( + "Failed to convert feature extractors.", + traceback.format_exc(), + ) + ) feature_extractors = [] if hasattr(tiny_config, "max_position_embeddings") and tiny_config.max_position_embeddings > 0: @@ -538,9 +614,9 @@ def build_model(model_arch, tiny_config, output_dir): return model -def fill_result_with_error(result, error, models_to_create): +def fill_result_with_error(result, error, trace, models_to_create): """Fill `result` with errors for all target model arch if we can't build processor""" - + error = (error, trace) result["error"] = error for framework in FRAMEWORKS: if framework in models_to_create: @@ -548,7 +624,7 @@ def fill_result_with_error(result, error, models_to_create): for model_arch in models_to_create[framework]: result[framework][model_arch.__name__] = {"model": None, "checkpoint": None, "error": error} - result["processor"] = {type(p).__name__: p.__class__.__name__ for p in result["processor"]} + result["processor"] = {p.__class__.__name__: p.__class__.__name__ for p in result["processor"].values()} def upload_model(model_dir, organization): @@ -572,7 +648,7 @@ def upload_model(model_dir, organization): except Exception as e: error = e if error is not None: - raise ValueError(error) + raise error with tempfile.TemporaryDirectory() as tmpdir: repo = Repository(local_dir=tmpdir, clone_from=f"{organization}/{repo_name}") @@ -589,13 +665,13 @@ def upload_model(model_dir, organization): commit_description=f"Upload tiny models for {arch_name}", create_pr=True, ) - logger.warning(f"PR open in {hub_pr_url}") + logger.warning(f"PR open in {hub_pr_url}.") else: # Push to Hub repo directly repo.git_add(auto_lfs_track=True) repo.git_commit(f"Upload tiny models for {arch_name}") repo.git_push(blocking=True) # this prints a progress bar with the upload - logger.warning(f"Tiny models {arch_name} pushed to {organization}/{repo_name}") + logger.warning(f"Tiny models {arch_name} pushed to {organization}/{repo_name}.") def build_composite_models(config_class, output_dir): @@ -715,7 +791,7 @@ def build_composite_models(config_class, output_dir): shutil.copytree(decoder_processor_path, model_path, dirs_exist_ok=True) # fill `result` - result["processor"] = tuple({x.__name__ for x in encoder_processor + decoder_processor}) + result["processor"] = {x.__name__: x.__name__ for x in encoder_processor + decoder_processor} result["pytorch"] = {model_class.__name__: {"model": model_class.__name__, "checkpoint": model_path}} @@ -724,9 +800,11 @@ def build_composite_models(config_class, output_dir): result["tensorflow"] = { tf_model_class.__name__: {"model": tf_model_class.__name__, "checkpoint": model_path} } - - except Exception as e: - result["error"] = f"Failed to build models for {config_class.__name__}: {e}" + except Exception: + result["error"] = ( + f"Failed to build models for {config_class.__name__}.", + traceback.format_exc(), + ) if not result["error"]: del result["error"] @@ -862,8 +940,8 @@ def build(config_class, models_to_create, output_dir): if len(processor_classes) == 0: error = f"No processor class could be found in {config_class.__name__}." - fill_result_with_error(result, error, models_to_create) - logger.error(result["error"]) + fill_result_with_error(result, error, None, models_to_create) + logger.error(result["error"][0]) return result for processor_class in processor_classes: @@ -871,24 +949,26 @@ def build(config_class, models_to_create, output_dir): processor = build_processor(config_class, processor_class, allow_no_checkpoint=True) if processor is not None: result["processor"][processor_class] = processor - except Exception as e: - error = f"Failed to build processor for {processor_class.__name__}: {e}" - fill_result_with_error(result, error, models_to_create) - logger.error(result["error"]) + except Exception: + error = f"Failed to build processor for {processor_class.__name__}." + trace = traceback.format_exc() + fill_result_with_error(result, error, trace, models_to_create) + logger.error(result["error"][0]) return result if len(result["processor"]) == 0: error = f"No processor could be built for {config_class.__name__}." - fill_result_with_error(result, error, models_to_create) - logger.error(result["error"]) + fill_result_with_error(result, error, None, models_to_create) + logger.error(result["error"][0]) return result try: tiny_config = get_tiny_config(config_class) except Exception as e: error = f"Failed to get tiny config for {config_class.__name__}: {e}" - fill_result_with_error(result, error, models_to_create) - logger.error(result["error"]) + trace = traceback.format_exc() + fill_result_with_error(result, error, trace, models_to_create) + logger.error(result["error"][0]) return result # Convert the processors (reduce vocabulary size, smaller image size, etc.) @@ -896,22 +976,24 @@ def build(config_class, models_to_create, output_dir): processor_output_folder = os.path.join(output_dir, "processors") try: processors = convert_processors(processors, tiny_config, processor_output_folder, result) - except Exception as e: - error = f"Failed to convert the processors: {e}" - result["warnings"].append(error) + except Exception: + error = "Failed to convert the processors." + trace = traceback.format_exc() + result["warnings"].append((error, trace)) if len(processors) == 0: error = f"No processor is returned by `convert_processors` for {config_class.__name__}." - fill_result_with_error(result, error, models_to_create) - logger.error(result["error"]) + fill_result_with_error(result, error, None, models_to_create) + logger.error(result["error"][0]) return result try: config_overrides = get_config_overrides(config_class, processors) except Exception as e: error = f"Failure occurs while calling `get_config_overrides`: {e}" - fill_result_with_error(result, error, models_to_create) - logger.error(result["error"]) + trace = traceback.format_exc() + fill_result_with_error(result, error, trace, models_to_create) + logger.error(result["error"][0]) return result # Just for us to see this easily in the report @@ -935,7 +1017,7 @@ def build(config_class, models_to_create, output_dir): tiny_config.text_config_dict[k] = v if result["warnings"]: - logger.warning(result["warnings"]) + logger.warning(result["warnings"][0][0]) # update `result["processor"]` result["processor"] = {type(p).__name__: p.__class__.__name__ for p in processors} @@ -948,13 +1030,14 @@ def build(config_class, models_to_create, output_dir): except Exception as e: model = None error = f"Failed to create the pytorch model for {pytorch_arch}: {e}" + trace = traceback.format_exc() result["pytorch"][pytorch_arch.__name__]["model"] = model.__class__.__name__ if model is not None else None result["pytorch"][pytorch_arch.__name__]["checkpoint"] = ( get_checkpoint_dir(output_dir, pytorch_arch) if model is not None else None ) if error is not None: - result["pytorch"][pytorch_arch.__name__]["error"] = error + result["pytorch"][pytorch_arch.__name__]["error"] = (error, trace) logger.error(f"{pytorch_arch.__name__}: {error}") for tensorflow_arch in models_to_create["tensorflow"]: @@ -974,12 +1057,14 @@ def build(config_class, models_to_create, output_dir): # Conversion may fail. Let's not create a model with different weights to avoid confusion (for now). model = None error = f"Failed to convert the pytorch model to the tensorflow model for {pt_arch}: {e}" + trace = traceback.format_exc() else: try: model = build_model(tensorflow_arch, tiny_config, output_dir=output_dir) except Exception as e: model = None error = f"Failed to create the tensorflow model for {tensorflow_arch}: {e}" + trace = traceback.format_exc() result["tensorflow"][tensorflow_arch.__name__]["model"] = ( model.__class__.__name__ if model is not None else None @@ -988,7 +1073,7 @@ def build(config_class, models_to_create, output_dir): get_checkpoint_dir(output_dir, tensorflow_arch) if model is not None else None ) if error is not None: - result["tensorflow"][tensorflow_arch.__name__]["error"] = error + result["tensorflow"][tensorflow_arch.__name__]["error"] = (error, trace) logger.error(f"{tensorflow_arch.__name__}: {error}") if not result["error"]: @@ -999,6 +1084,37 @@ def build(config_class, models_to_create, output_dir): return result +def build_tiny_model_summary(results): + """Build a summary: a dictionary of the form + { + model architecture name: + { + "tokenizer_classes": [...], + "processor_classes": [...] + } + .. + } + """ + tiny_model_summary = {} + for config_name in results: + processors = [key for key, value in results[config_name]["processor"].items()] + tokenizer_classes = [x for x in processors if x.endswith("TokenizerFast") or x.endswith("Tokenizer")] + processor_classes = [x for x in processors if x not in tokenizer_classes] + for framework in FRAMEWORKS: + if framework not in results[config_name]: + continue + for arch_name in results[config_name][framework]: + # tiny model is not created for `arch_name` + if results[config_name][framework][arch_name] is None: + continue + tiny_model_summary[arch_name] = { + "tokenizer_classes": tokenizer_classes, + "processor_classes": processor_classes, + } + + return tiny_model_summary + + def build_failed_report(results, include_warning=True): failed_results = {} for config_name in results: @@ -1039,10 +1155,10 @@ def build_simple_report(results): for arch_name in results[config_name][framework]: if "error" in results[config_name][framework][arch_name]: result = results[config_name][framework][arch_name]["error"] - failed_text += f"{arch_name}: {result}\n" + failed_text += f"{arch_name}: {result[0]}\n" else: - result = "OK" - text += f"{arch_name}: {result}\n" + result = ("OK",) + text += f"{arch_name}: {result[0]}\n" return text, failed_text @@ -1066,8 +1182,6 @@ def build_simple_report(results): tensorflow_arch_mappings = [getattr(transformers_module, x) for x in _tensorflow_arch_mappings] # flax_arch_mappings = [getattr(transformers_module, x) for x in _flax_arch_mappings] - unexportable_model_architectures = [] - ds = load_dataset("wikitext", "wikitext-2-raw-v1") training_ds = ds["train"] testing_ds = ds["test"] @@ -1129,16 +1243,27 @@ def list_str(values): with open("tiny_model_creation_report.json", "w") as fp: json.dump(results, fp, indent=4) - # Build the failure report + # Build the tiny model summary file. The `tokenizer_classes` and `processor_classes` could be both empty lists. + # When using the items in this file to update the file `tests/utils/tiny_model_summary.json`, the model + # architectures with `tokenizer_classes` and `processor_classes` being both empty should **NOT** be added to + # `tests/utils/tiny_model_summary.json`. + tiny_model_summary = build_tiny_model_summary(results) + with open("tiny_model_summary.json", "w") as fp: + json.dump(tiny_model_summary, fp, indent=4) + + # Build the warning/failure report (json format): same format as the complete `results` except this contains only + # warnings or errors. failed_results = build_failed_report(results) with open("failed_report.json", "w") as fp: json.dump(failed_results, fp, indent=4) - # Build the failure report simple_report, failed_report = build_simple_report(results) + # The simplified report: a .txt file with each line of format: + # {model architecture name}: {OK or error message} with open("simple_report.txt", "w") as fp: fp.write(simple_report) + # The simplified failure report: same above except this only contains line with errors with open("simple_failed_report.txt", "w") as fp: fp.write(failed_report) @@ -1160,7 +1285,7 @@ def list_str(values): try: upload_model(model_dir, args.organization) except Exception as e: - error = f"Failed to upload {model_dir}: {e}" + error = f"Failed to upload {model_dir}. {e.__class__.__name__}: {e}" logger.error(error) upload_results[model_dir] = error From 777f921775c9f24c29f43cf104c9625dd6c1965b Mon Sep 17 00:00:00 2001 From: aws-sangeetha <83724701+aws-sangeetha@users.noreply.github.com> Date: Thu, 9 Mar 2023 11:31:58 -0800 Subject: [PATCH 050/277] Fix case when using --gradient_accumulation_steps with DDP disabled. (#22007) Co-authored-by: EC2 Default User --- src/transformers/training_args.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index 67a796e713e6..32b1f0059063 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -1801,7 +1801,9 @@ def _no_sync_in_gradient_accumulation(self): """ Whether or not to use no_sync for the gradients when doing gradient accumulation. """ - return not (self.deepspeed or is_sagemaker_dp_enabled() or is_sagemaker_mp_enabled()) + return not ( + self.deepspeed or is_sagemaker_dp_enabled() or is_sagemaker_mp_enabled() or is_torch_neuroncore_available() + ) @contextlib.contextmanager def main_process_first(self, local=True, desc="work"): From 1e9ccb61c18bf647369279f9420f44cc887d0619 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:58:03 -0500 Subject: [PATCH 051/277] Add a progress bar for the total download of shards (#22062) * Add a progress bar for the total download of shards * Check for no cache at all * Fix check --- src/transformers/models/detr/configuration_detr.py | 1 + src/transformers/utils/hub.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/detr/configuration_detr.py b/src/transformers/models/detr/configuration_detr.py index b3da5f86b016..955b71de1ec5 100644 --- a/src/transformers/models/detr/configuration_detr.py +++ b/src/transformers/models/detr/configuration_detr.py @@ -239,6 +239,7 @@ def hidden_size(self) -> int: @classmethod def from_backbone_config(cls, backbone_config: PretrainedConfig, **kwargs): """Instantiate a [`DetrConfig`] (or a derived class) from a pre-trained backbone model configuration. + Args: backbone_config ([`PretrainedConfig`]): The backbone configuration. diff --git a/src/transformers/utils/hub.py b/src/transformers/utils/hub.py index 3403867eafe8..db00878c9ae4 100644 --- a/src/transformers/utils/hub.py +++ b/src/transformers/utils/hub.py @@ -390,7 +390,7 @@ def cached_file( if isinstance(cache_dir, Path): cache_dir = str(cache_dir) - if _commit_hash is not None: + if _commit_hash is not None and not force_download: # If the file is cached under that commit hash, we return it directly. resolved_file = try_to_load_from_cache( path_or_repo_id, full_filename, cache_dir=cache_dir, revision=_commit_hash @@ -913,7 +913,13 @@ def get_checkpoint_shard_files( # At this stage pretrained_model_name_or_path is a model identifier on the Hub cached_filenames = [] - for shard_filename in shard_filenames: + # Check if the model is already cached or not. We only try the last checkpoint, this should cover most cases of + # downloaded (if interrupted). + last_shard = try_to_load_from_cache( + pretrained_model_name_or_path, shard_filenames[-1], cache_dir=cache_dir, revision=_commit_hash + ) + show_progress_bar = last_shard is None or force_download + for shard_filename in tqdm(shard_filenames, desc="Downloading shards", disable=not show_progress_bar): try: # Load from URL cached_filename = cached_file( From b84287ee97a4d5db73095b4171d3a2055e431477 Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Fri, 10 Mar 2023 12:30:42 +0100 Subject: [PATCH 052/277] Fix gradient checkpointing bug in Speech2Text (#22079) * Fix gradient checkpointing bug in Speech2Text * Update modeling_speech_to_text.py * Update modeling_speech_to_text_2.py --- .../speech_to_text/modeling_speech_to_text.py | 13 +++++++------ .../speech_to_text_2/modeling_speech_to_text_2.py | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/transformers/models/speech_to_text/modeling_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_speech_to_text.py index eaef470e0822..a8c1a640f2ff 100755 --- a/src/transformers/models/speech_to_text/modeling_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_speech_to_text.py @@ -1024,6 +1024,13 @@ def forward( hidden_states = inputs_embeds + positions hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache =" " False`..." + ) + use_cache = False + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -1048,12 +1055,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning( - "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache =" - " False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): diff --git a/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py b/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py index e408974a5053..319589eab144 100755 --- a/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py +++ b/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py @@ -632,6 +632,13 @@ def forward( hidden_states = inputs_embeds + positions hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache =" " False`..." + ) + use_cache = False + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -657,12 +664,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning( - "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache =" - " False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From ae930f0bf5b036800e2b8792949a183ba7710ed2 Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Fri, 10 Mar 2023 12:31:08 +0100 Subject: [PATCH 053/277] Fix gradient checkpointing bug in switch transformer (#22081) --- .../modeling_switch_transformers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/switch_transformers/modeling_switch_transformers.py b/src/transformers/models/switch_transformers/modeling_switch_transformers.py index de24797c67b0..bcf1c4b7bc83 100644 --- a/src/transformers/models/switch_transformers/modeling_switch_transformers.py +++ b/src/transformers/models/switch_transformers/modeling_switch_transformers.py @@ -1039,6 +1039,13 @@ def forward( else: encoder_extended_attention_mask = None + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + # Prepare head mask if needed head_mask = self.get_head_mask(head_mask, self.config.num_layers) cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers) @@ -1060,11 +1067,6 @@ def forward( all_hidden_states = all_hidden_states + (hidden_states,) if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 3d42b725fe357d45fe4f745e1bf700a09f06c1cc Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Fri, 10 Mar 2023 13:15:25 +0100 Subject: [PATCH 054/277] [GPT2] Propose fix for #21080 (#21853) * Make sure position ids are masked * test that padded input produce the same results * fix failing tests * fixup * fix batch test --- .../modeling_decision_transformer.py | 9 +++++++- src/transformers/models/gpt2/modeling_gpt2.py | 9 +++++++- tests/models/gpt2/test_modeling_gpt2.py | 21 +++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/decision_transformer/modeling_decision_transformer.py b/src/transformers/models/decision_transformer/modeling_decision_transformer.py index 0add713d8e87..d489be4f2d3f 100755 --- a/src/transformers/models/decision_transformer/modeling_decision_transformer.py +++ b/src/transformers/models/decision_transformer/modeling_decision_transformer.py @@ -553,7 +553,14 @@ def forward( past_key_values = tuple([None] * len(self.h)) else: past_length = past_key_values[0][0].size(-2) - if position_ids is None: + + if attention_mask is not None and len(attention_mask.shape) == 2 and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_length > 0: + position_ids = position_ids[:, past_length : input_shape[-1] + past_length :] + elif position_ids is None: position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) diff --git a/src/transformers/models/gpt2/modeling_gpt2.py b/src/transformers/models/gpt2/modeling_gpt2.py index bf6651431480..03a1a3c02e0b 100644 --- a/src/transformers/models/gpt2/modeling_gpt2.py +++ b/src/transformers/models/gpt2/modeling_gpt2.py @@ -797,7 +797,14 @@ def forward( past_key_values = tuple([None] * len(self.h)) else: past_length = past_key_values[0][0].size(-2) - if position_ids is None: + + if attention_mask is not None and len(attention_mask.shape) == 2 and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_length > 0: + position_ids = position_ids[:, past_length : input_shape[-1] + past_length :] + elif position_ids is None: position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) diff --git a/tests/models/gpt2/test_modeling_gpt2.py b/tests/models/gpt2/test_modeling_gpt2.py index 09d828fd7f33..be5445525b4e 100644 --- a/tests/models/gpt2/test_modeling_gpt2.py +++ b/tests/models/gpt2/test_modeling_gpt2.py @@ -590,6 +590,27 @@ def test_batch_generation(self): self.assertTrue(batch_out_sentence_tt != batch_out_sentence) # token_type_ids should change output self.assertListEqual(expected_output_sentence, [non_padded_sentence, padded_sentence]) + @slow + def test_batch_forward(self): + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.padding_side = "left" + + # This tokenizer has no pad token, so we have to set it in some way + # Define PAD Token = EOS Token = 50256 + tokenizer.pad_token = tokenizer.eos_token + + model = GPT2LMHeadModel.from_pretrained("gpt2", pad_token_id=tokenizer.eos_token_id) + sentences = ["Hello, my dog is a little bit of a mess. I'm not sure if he's"] + inputs = tokenizer(sentences, padding=True, return_tensors="pt") + logits = model(**inputs).logits[:, -1, :] + indexes = torch.argmax(logits).item() + + inputs_padded = tokenizer(sentences, padding="max_length", max_length=30, return_tensors="pt") + logits_padded = model(**inputs_padded).logits[:, -1, :] + indexes_padded = torch.argmax(logits_padded).item() + + self.assertTrue(indexes == indexes_padded) + @slow def test_batch_generation_2heads(self): model = GPT2DoubleHeadsModel.from_pretrained("gpt2") From 13289d4b28654486afb0fbe63d946a874c967950 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Fri, 10 Mar 2023 07:44:45 -0500 Subject: [PATCH 055/277] Fix small typo in flan-ul2.mdx (#22068) * Update flan-ul2.mdx * Update flan-ul2.mdx --- docs/source/en/model_doc/flan-ul2.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/flan-ul2.mdx b/docs/source/en/model_doc/flan-ul2.mdx index d1687b2be261..bcc00295420e 100644 --- a/docs/source/en/model_doc/flan-ul2.mdx +++ b/docs/source/en/model_doc/flan-ul2.mdx @@ -31,7 +31,7 @@ One can refer to [T5's documentation page](t5) for all tips, code examples and n The original checkpoints can be found [here](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints). -## Running on low ressources devices +## Running on low resource devices The model is pretty heavy (~40GB in half precision) so if you just want to run the model, make sure you load your model in 8bit, and use `device_map="auto"` to make sure you don't have any OOM issue! @@ -49,4 +49,4 @@ The model is pretty heavy (~40GB in half precision) so if you just want to run t ## Inference -The inference protocol is exaclty the same as any `T5` model, please have a look at the [T5's documentation page](t5) for more details. \ No newline at end of file +The inference protocol is exaclty the same as any `T5` model, please have a look at the [T5's documentation page](t5) for more details. From c78d87a3a4d93c2912876437ca50b82fe515f74a Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Fri, 10 Mar 2023 13:28:30 +0000 Subject: [PATCH 056/277] Generate - Fix broken documentation links (#22078) fix broken links --- src/transformers/generation/flax_utils.py | 2 +- src/transformers/generation/tf_utils.py | 4 ++-- src/transformers/generation/utils.py | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/transformers/generation/flax_utils.py b/src/transformers/generation/flax_utils.py index 440a0ae26275..4ff1164c88e9 100644 --- a/src/transformers/generation/flax_utils.py +++ b/src/transformers/generation/flax_utils.py @@ -140,7 +140,7 @@ class FlaxGenerationMixin: `do_sample=False` You do not need to call any of the above methods directly. Pass custom parameter values to 'generate' instead. To - learn more about decoding strategies refer to the [text generation strategies guide](./generation_strategies). + learn more about decoding strategies refer to the [text generation strategies guide](../generation_strategies). """ def prepare_inputs_for_generation(self, *args, **kwargs): diff --git a/src/transformers/generation/tf_utils.py b/src/transformers/generation/tf_utils.py index 2cb9c7cad709..4a9140f8853d 100644 --- a/src/transformers/generation/tf_utils.py +++ b/src/transformers/generation/tf_utils.py @@ -455,7 +455,7 @@ class TFGenerationMixin: - *beam-search decoding* by calling [`~generation.TFGenerationMixin.beam_search`] if `num_beams>1` You do not need to call any of the above methods directly. Pass custom parameter values to 'generate' instead. To - learn more about decoding strategies refer to the [text generation strategies guide](./generation_strategies). + learn more about decoding strategies refer to the [text generation strategies guide](../generation_strategies). """ _seed_generator = None @@ -681,7 +681,7 @@ def generate( parameters to generate, e.g. `.generate(inputs, num_beams=4, do_sample=True)`. For an overview of generation strategies and code examples, check out the [following - guide](./generation_strategies). + guide](../generation_strategies). diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 0624550493b5..7091b49e3ecc 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -483,7 +483,7 @@ class GenerationMixin: `constraints!=None` or `force_words_ids!=None` You do not need to call any of the above methods directly. Pass custom parameter values to 'generate' instead. To - learn more about decoding strategies refer to the [text generation strategies guide](./generation_strategies). + learn more about decoding strategies refer to the [text generation strategies guide](../generation_strategies). """ def prepare_inputs_for_generation(self, *args, **kwargs): @@ -1129,7 +1129,7 @@ def generate( parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`. For an overview of generation strategies and code examples, check out the [following - guide](./generation_strategies). + guide](../generation_strategies). @@ -1699,7 +1699,7 @@ def contrastive_search( In most cases, you do not need to call [`~generation.GenerationMixin.contrastive_search`] directly. Use generate() instead. For an overview of generation strategies and code examples, check the [following - guide](./generation_strategies). + guide](../generation_strategies). @@ -2057,7 +2057,7 @@ def greedy_search( In most cases, you do not need to call [`~generation.GenerationMixin.greedy_search`] directly. Use generate() instead. For an overview of generation strategies and code examples, check the [following - guide](./generation_strategies). + guide](../generation_strategies). @@ -2304,7 +2304,7 @@ def sample( In most cases, you do not need to call [`~generation.GenerationMixin.sample`] directly. Use generate() instead. For an overview of generation strategies and code examples, check the [following - guide](./generation_strategies). + guide](../generation_strategies). @@ -2573,7 +2573,7 @@ def beam_search( In most cases, you do not need to call [`~generation.GenerationMixin.beam_search`] directly. Use generate() instead. For an overview of generation strategies and code examples, check the [following - guide](./generation_strategies). + guide](../generation_strategies). @@ -2897,7 +2897,7 @@ def beam_sample( In most cases, you do not need to call [`~generation.GenerationMixin.beam_sample`] directly. Use generate() instead. For an overview of generation strategies and code examples, check the [following - guide](./generation_strategies). + guide](../generation_strategies). @@ -3229,7 +3229,7 @@ def group_beam_search( In most cases, you do not need to call [`~generation.GenerationMixin.group_beam_search`] directly. Use generate() instead. For an overview of generation strategies and code examples, check the [following - guide](./generation_strategies). + guide](../generation_strategies). @@ -3607,7 +3607,7 @@ def constrained_beam_search( In most cases, you do not need to call [`~generation.GenerationMixin.constrained_beam_search`] directly. Use generate() instead. For an overview of generation strategies and code examples, check the [following - guide](./generation_strategies). + guide](../generation_strategies). From 0993014b6e162884a4028b665dbd5703d7ea11f0 Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:36:09 +0100 Subject: [PATCH 057/277] Fix gradient checkpointing bug in Speecht5 (#22080) * Fix gradient checkpointing bug in Speecht5 * Update modeling_speech_to_text.py * Update src/transformers/models/speech_to_text/modeling_speech_to_text.py * Fix change errors --------- Co-authored-by: Joao Gante --- .../models/speech_to_text/modeling_speech_to_text.py | 2 +- .../models/speecht5/modeling_speecht5.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/speech_to_text/modeling_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_speech_to_text.py index a8c1a640f2ff..d08863f8353f 100755 --- a/src/transformers/models/speech_to_text/modeling_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_speech_to_text.py @@ -1027,7 +1027,7 @@ def forward( if self.gradient_checkpointing and self.training: if use_cache: logger.warning_once( - "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache =" " False`..." + "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..." ) use_cache = False diff --git a/src/transformers/models/speecht5/modeling_speecht5.py b/src/transformers/models/speecht5/modeling_speecht5.py index e67c55c23b4e..975f483395be 100644 --- a/src/transformers/models/speecht5/modeling_speecht5.py +++ b/src/transformers/models/speecht5/modeling_speecht5.py @@ -1662,6 +1662,13 @@ def forward( deepspeed_zero3_is_enabled = is_deepspeed_zero3_enabled() + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None @@ -1691,11 +1698,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 9199bf7224328e201394fd8347b4a89a30e119e1 Mon Sep 17 00:00:00 2001 From: J-shang <33053116+J-shang@users.noreply.github.com> Date: Fri, 10 Mar 2023 21:56:42 +0800 Subject: [PATCH 058/277] Fix hint in src/transformers/modeling_utils.py (#22074) fix hint --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 494367b813bf..e85d8e304d09 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -807,7 +807,7 @@ def create_extended_attention_mask_for_decoder(input_shape, attention_mask, devi return extended_attention_mask def get_extended_attention_mask( - self, attention_mask: Tensor, input_shape: Tuple[int], device: device = None, dtype: torch.float = None + self, attention_mask: Tensor, input_shape: Tuple[int], device: torch.device = None, dtype: torch.float = None ) -> Tensor: """ Makes broadcastable attention and causal masks so that future and masked tokens are ignored. From 6a42232da2f6a0da744d6a2baecbcb57a1f4c0fb Mon Sep 17 00:00:00 2001 From: Dean Wyatte <2512762+dwyatte@users.noreply.github.com> Date: Fri, 10 Mar 2023 08:50:29 -0700 Subject: [PATCH 059/277] handle numpy inputs in whole word mask data collator (#22032) --- src/transformers/data/data_collator.py | 4 +++- tests/trainer/test_data_collator.py | 27 ++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py index 587a4f4d0043..cd363588757b 100644 --- a/src/transformers/data/data_collator.py +++ b/src/transformers/data/data_collator.py @@ -883,6 +883,8 @@ def torch_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> D return {"input_ids": inputs, "labels": labels} def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: + import tensorflow as tf + if isinstance(examples[0], Mapping): input_ids = [e["input_ids"] for e in examples] else: @@ -907,7 +909,7 @@ def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict ref_tokens[i] = "##" + ref_tokens[i] mask_labels.append(self._whole_word_mask(ref_tokens)) batch_mask = _tf_collate_batch(mask_labels, self.tokenizer, pad_to_multiple_of=self.pad_to_multiple_of) - inputs, labels = self.tf_mask_tokens(batch_input, batch_mask) + inputs, labels = self.tf_mask_tokens(tf.cast(batch_input, tf.int64), batch_mask) return {"input_ids": inputs, "labels": labels} def numpy_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]]) -> Dict[str, Any]: diff --git a/tests/trainer/test_data_collator.py b/tests/trainer/test_data_collator.py index 39277ca8cc18..f5104cd37507 100644 --- a/tests/trainer/test_data_collator.py +++ b/tests/trainer/test_data_collator.py @@ -271,12 +271,17 @@ def test_data_collator_for_language_modeling(self): self._test_no_pad_and_pad(no_pad_features, pad_features) def test_data_collator_for_whole_word_mask(self): - features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}] - tokenizer = BertTokenizer(self.vocab_file) data_collator = DataCollatorForWholeWordMask(tokenizer, return_tensors="pt") + + features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}] batch = data_collator(features) + self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10))) + self.assertEqual(batch["labels"].shape, torch.Size((2, 10))) + # Features can already be tensors + features = [{"input_ids": np.arange(10)}, {"input_ids": np.arange(10)}] + batch = data_collator(features) self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10))) self.assertEqual(batch["labels"].shape, torch.Size((2, 10))) @@ -553,12 +558,17 @@ def test_data_collator_for_language_modeling(self): self._test_no_pad_and_pad(no_pad_features, pad_features) def test_data_collator_for_whole_word_mask(self): - features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}] - tokenizer = BertTokenizer(self.vocab_file) data_collator = DataCollatorForWholeWordMask(tokenizer, return_tensors="tf") + + features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}] batch = data_collator(features) + self.assertEqual(batch["input_ids"].shape.as_list(), [2, 10]) + self.assertEqual(batch["labels"].shape.as_list(), [2, 10]) + # Features can already be tensors + features = [{"input_ids": np.arange(10)}, {"input_ids": np.arange(10)}] + batch = data_collator(features) self.assertEqual(batch["input_ids"].shape.as_list(), [2, 10]) self.assertEqual(batch["labels"].shape.as_list(), [2, 10]) @@ -825,12 +835,17 @@ def test_data_collator_for_language_modeling(self): self._test_no_pad_and_pad(no_pad_features, pad_features) def test_data_collator_for_whole_word_mask(self): - features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}] - tokenizer = BertTokenizer(self.vocab_file) data_collator = DataCollatorForWholeWordMask(tokenizer, return_tensors="np") + + features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}] batch = data_collator(features) + self.assertEqual(batch["input_ids"].shape, (2, 10)) + self.assertEqual(batch["labels"].shape, (2, 10)) + # Features can already be tensors + features = [{"input_ids": np.arange(10)}, {"input_ids": np.arange(10)}] + batch = data_collator(features) self.assertEqual(batch["input_ids"].shape, (2, 10)) self.assertEqual(batch["labels"].shape, (2, 10)) From 18fa8e9901e6f4b824eacfd0647ba35be51b64bd Mon Sep 17 00:00:00 2001 From: Maria Khalusova Date: Fri, 10 Mar 2023 14:03:43 -0500 Subject: [PATCH 060/277] GPT-J specific half precision on CPU note (#22086) * re: #21989 * update re: #21989 * removed cpu option * make style --- docs/source/en/model_doc/gptj.mdx | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/source/en/model_doc/gptj.mdx b/docs/source/en/model_doc/gptj.mdx index 98247fcfb044..2b407d7b2d40 100644 --- a/docs/source/en/model_doc/gptj.mdx +++ b/docs/source/en/model_doc/gptj.mdx @@ -21,21 +21,22 @@ This model was contributed by [Stella Biderman](https://huggingface.co/stellaath Tips: -- To load [GPT-J](https://huggingface.co/EleutherAI/gpt-j-6B) in float32 one would need at least 2x model size CPU - RAM: 1x for initial weights and another 1x to load the checkpoint. So for GPT-J it would take at least 48GB of CPU - RAM to just load the model. To reduce the CPU RAM usage there are a few options. The `torch_dtype` argument can be - used to initialize the model in half-precision. And the `low_cpu_mem_usage` argument can be used to keep the RAM - usage to 1x. There is also a [fp16 branch](https://huggingface.co/EleutherAI/gpt-j-6B/tree/float16) which stores - the fp16 weights, which could be used to further minimize the RAM usage. Combining all this it should take roughly - 12.1GB of CPU RAM to load the model. +- To load [GPT-J](https://huggingface.co/EleutherAI/gpt-j-6B) in float32 one would need at least 2x model size + RAM: 1x for initial weights and another 1x to load the checkpoint. So for GPT-J it would take at least 48GB + RAM to just load the model. To reduce the RAM usage there are a few options. The `torch_dtype` argument can be + used to initialize the model in half-precision on a CUDA device only. There is also a fp16 branch which stores the fp16 weights, + which could be used to further minimize the RAM usage: ```python >>> from transformers import GPTJForCausalLM >>> import torch +>>> device = "cuda" >>> model = GPTJForCausalLM.from_pretrained( -... "EleutherAI/gpt-j-6B", revision="float16", torch_dtype=torch.float16, low_cpu_mem_usage=True -... ) +... "EleutherAI/gpt-j-6B", +... revision="float16", +... torch_dtype=torch.float16, +... ).to(device) ``` - The model should fit on 16GB GPU for inference. For training/fine-tuning it would take much more GPU RAM. Adam @@ -85,7 +86,8 @@ model. >>> from transformers import GPTJForCausalLM, AutoTokenizer >>> import torch ->>> model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B", torch_dtype=torch.float16) +>>> device = "cuda" +>>> model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B", torch_dtype=torch.float16).to(device) >>> tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6B") >>> prompt = ( @@ -94,7 +96,7 @@ model. ... "researchers was the fact that the unicorns spoke perfect English." ... ) ->>> input_ids = tokenizer(prompt, return_tensors="pt").input_ids +>>> input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to(device) >>> gen_tokens = model.generate( ... input_ids, From 9fd5b5ee60fc3acd2c319f727f06116639e9df35 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:46:34 -0500 Subject: [PATCH 061/277] Fix imports of TF MobileViT (#22065) * Fix imports of TF MobileViT * Fix copies --- src/transformers/__init__.py | 4 +++- src/transformers/utils/dummy_tf_objects.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 19ff7a1f83af..2438807378a3 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -6223,7 +6223,6 @@ from .models.mbart import TFMBartForConditionalGeneration, TFMBartModel, TFMBartPreTrainedModel from .models.mobilebert import ( TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST, - TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST, TFMobileBertForMaskedLM, TFMobileBertForMultipleChoice, TFMobileBertForNextSentencePrediction, @@ -6234,6 +6233,9 @@ TFMobileBertMainLayer, TFMobileBertModel, TFMobileBertPreTrainedModel, + ) + from .models.mobilevit import ( + TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST, TFMobileViTForImageClassification, TFMobileViTForSemanticSegmentation, TFMobileViTModel, diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index 3eac414edd5d..16d2e6820c16 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -1647,9 +1647,6 @@ def __init__(self, *args, **kwargs): TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None -TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST = None - - class TFMobileBertForMaskedLM(metaclass=DummyObject): _backends = ["tf"] @@ -1720,6 +1717,9 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) +TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST = None + + class TFMobileViTForImageClassification(metaclass=DummyObject): _backends = ["tf"] From ffddc14f6577759c64fa9c9f1e0c5be896c4bd99 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 10 Mar 2023 22:08:21 +0100 Subject: [PATCH 062/277] Revert "[GPT2] Propose fix for #21080" (#22093) Revert "[GPT2] Propose fix for #21080 (#21853)" to avoid CI failure This reverts commit a3fef89b2694fac4dd642a3f77d3e96d0c3df82a. --- .../modeling_decision_transformer.py | 9 +------- src/transformers/models/gpt2/modeling_gpt2.py | 9 +------- tests/models/gpt2/test_modeling_gpt2.py | 21 ------------------- 3 files changed, 2 insertions(+), 37 deletions(-) diff --git a/src/transformers/models/decision_transformer/modeling_decision_transformer.py b/src/transformers/models/decision_transformer/modeling_decision_transformer.py index d489be4f2d3f..0add713d8e87 100755 --- a/src/transformers/models/decision_transformer/modeling_decision_transformer.py +++ b/src/transformers/models/decision_transformer/modeling_decision_transformer.py @@ -553,14 +553,7 @@ def forward( past_key_values = tuple([None] * len(self.h)) else: past_length = past_key_values[0][0].size(-2) - - if attention_mask is not None and len(attention_mask.shape) == 2 and position_ids is None: - # create position_ids on the fly for batch generation - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - if past_length > 0: - position_ids = position_ids[:, past_length : input_shape[-1] + past_length :] - elif position_ids is None: + if position_ids is None: position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) diff --git a/src/transformers/models/gpt2/modeling_gpt2.py b/src/transformers/models/gpt2/modeling_gpt2.py index 03a1a3c02e0b..bf6651431480 100644 --- a/src/transformers/models/gpt2/modeling_gpt2.py +++ b/src/transformers/models/gpt2/modeling_gpt2.py @@ -797,14 +797,7 @@ def forward( past_key_values = tuple([None] * len(self.h)) else: past_length = past_key_values[0][0].size(-2) - - if attention_mask is not None and len(attention_mask.shape) == 2 and position_ids is None: - # create position_ids on the fly for batch generation - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - if past_length > 0: - position_ids = position_ids[:, past_length : input_shape[-1] + past_length :] - elif position_ids is None: + if position_ids is None: position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) diff --git a/tests/models/gpt2/test_modeling_gpt2.py b/tests/models/gpt2/test_modeling_gpt2.py index be5445525b4e..09d828fd7f33 100644 --- a/tests/models/gpt2/test_modeling_gpt2.py +++ b/tests/models/gpt2/test_modeling_gpt2.py @@ -590,27 +590,6 @@ def test_batch_generation(self): self.assertTrue(batch_out_sentence_tt != batch_out_sentence) # token_type_ids should change output self.assertListEqual(expected_output_sentence, [non_padded_sentence, padded_sentence]) - @slow - def test_batch_forward(self): - tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - tokenizer.padding_side = "left" - - # This tokenizer has no pad token, so we have to set it in some way - # Define PAD Token = EOS Token = 50256 - tokenizer.pad_token = tokenizer.eos_token - - model = GPT2LMHeadModel.from_pretrained("gpt2", pad_token_id=tokenizer.eos_token_id) - sentences = ["Hello, my dog is a little bit of a mess. I'm not sure if he's"] - inputs = tokenizer(sentences, padding=True, return_tensors="pt") - logits = model(**inputs).logits[:, -1, :] - indexes = torch.argmax(logits).item() - - inputs_padded = tokenizer(sentences, padding="max_length", max_length=30, return_tensors="pt") - logits_padded = model(**inputs_padded).logits[:, -1, :] - indexes_padded = torch.argmax(logits_padded).item() - - self.assertTrue(indexes == indexes_padded) - @slow def test_batch_generation_2heads(self): model = GPT2DoubleHeadsModel.from_pretrained("gpt2") From 30ccdfa8d0bb67eac3b311ad47e50fbc33f505c1 Mon Sep 17 00:00:00 2001 From: Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> Date: Sat, 11 Mar 2023 14:03:36 +0100 Subject: [PATCH 063/277] [Whisper] Remove embed_tokens from encoder docstring (#21996) * [Whisper] Remove embed_tokens from encoder docstring * new line to retrigger CI * remove new line --- src/transformers/models/whisper/modeling_whisper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index cefcac389507..94d0be40476d 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -737,7 +737,6 @@ class WhisperEncoder(WhisperPreTrainedModel): Args: config: WhisperConfig - embed_tokens (nn.Embedding): output embedding """ def __init__(self, config: WhisperConfig): From 733b396fc7614d122406821f545be2c989281b0f Mon Sep 17 00:00:00 2001 From: Alara Dirik <8944735+alaradirik@users.noreply.github.com> Date: Mon, 13 Mar 2023 12:46:14 +0300 Subject: [PATCH 064/277] Add AutoModelForZeroShotImageClassification (#22087) Adds AutoModelForZeroShotImageClassification to transformers --- docs/source/en/model_doc/auto.mdx | 8 +++++++ src/transformers/__init__.py | 8 +++++++ src/transformers/modelcard.py | 2 ++ src/transformers/models/auto/__init__.py | 8 +++++++ src/transformers/models/auto/modeling_auto.py | 14 ++++++++++++- .../models/auto/modeling_tf_auto.py | 21 +++++++++++++++++++ src/transformers/pipelines/__init__.py | 6 ++++-- .../zero_shot_image_classification.py | 14 +++++++++---- src/transformers/utils/dummy_pt_objects.py | 10 +++++++++ src/transformers/utils/dummy_tf_objects.py | 10 +++++++++ src/transformers/utils/fx.py | 2 ++ utils/update_metadata.py | 4 ++-- 12 files changed, 98 insertions(+), 9 deletions(-) diff --git a/docs/source/en/model_doc/auto.mdx b/docs/source/en/model_doc/auto.mdx index 9df4fa9c995d..39b0645eb5c8 100644 --- a/docs/source/en/model_doc/auto.mdx +++ b/docs/source/en/model_doc/auto.mdx @@ -258,6 +258,14 @@ The following auto classes are available for the following computer vision tasks [[autodoc]] AutoModelForUniversalSegmentation +### AutoModelForZeroShotImageClassification + +[[autodoc]] AutoModelForZeroShotImageClassification + +### TFAutoModelForZeroShotImageClassification + +[[autodoc]] TFAutoModelForZeroShotImageClassification + ### AutoModelForZeroShotObjectDetection [[autodoc]] AutoModelForZeroShotObjectDetection diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 2438807378a3..f32f3c417e94 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1001,6 +1001,7 @@ "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING", "MODEL_FOR_VISION_2_SEQ_MAPPING", "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING", + "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING", "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING", "MODEL_MAPPING", "MODEL_WITH_LM_HEAD_MAPPING", @@ -1033,6 +1034,7 @@ "AutoModelForVideoClassification", "AutoModelForVision2Seq", "AutoModelForVisualQuestionAnswering", + "AutoModelForZeroShotImageClassification", "AutoModelForZeroShotObjectDetection", "AutoModelWithLMHead", ] @@ -2785,6 +2787,7 @@ "TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING", "TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING", "TF_MODEL_FOR_VISION_2_SEQ_MAPPING", + "TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING", "TF_MODEL_MAPPING", "TF_MODEL_WITH_LM_HEAD_MAPPING", "TFAutoModel", @@ -2803,6 +2806,7 @@ "TFAutoModelForTableQuestionAnswering", "TFAutoModelForTokenClassification", "TFAutoModelForVision2Seq", + "TFAutoModelForZeroShotImageClassification", "TFAutoModelWithLMHead", ] ) @@ -4515,6 +4519,7 @@ MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, MODEL_FOR_VISION_2_SEQ_MAPPING, MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING, + MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING, MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING, MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, @@ -4547,6 +4552,7 @@ AutoModelForVideoClassification, AutoModelForVision2Seq, AutoModelForVisualQuestionAnswering, + AutoModelForZeroShotImageClassification, AutoModelForZeroShotObjectDetection, AutoModelWithLMHead, ) @@ -5972,6 +5978,7 @@ TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING, TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, TF_MODEL_FOR_VISION_2_SEQ_MAPPING, + TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING, TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING, TFAutoModel, @@ -5990,6 +5997,7 @@ TFAutoModelForTableQuestionAnswering, TFAutoModelForTokenClassification, TFAutoModelForVision2Seq, + TFAutoModelForZeroShotImageClassification, TFAutoModelWithLMHead, ) from .models.bart import ( diff --git a/src/transformers/modelcard.py b/src/transformers/modelcard.py index ac954272cda4..e89216b0d8b3 100644 --- a/src/transformers/modelcard.py +++ b/src/transformers/modelcard.py @@ -43,6 +43,7 @@ MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES, MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES, + MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES, ) from .training_args import ParallelMode from .utils import ( @@ -70,6 +71,7 @@ "token-classification": MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES, "audio-classification": MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES, "automatic-speech-recognition": {**MODEL_FOR_CTC_MAPPING_NAMES, **MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES}, + "zero-shot-image-classification": MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES, } logger = logging.get_logger(__name__) diff --git a/src/transformers/models/auto/__init__.py b/src/transformers/models/auto/__init__.py index 73965b657f3b..4eccfded5b6a 100644 --- a/src/transformers/models/auto/__init__.py +++ b/src/transformers/models/auto/__init__.py @@ -69,6 +69,7 @@ "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING", "MODEL_MAPPING", "MODEL_WITH_LM_HEAD_MAPPING", + "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING", "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING", "AutoModel", "AutoBackbone", @@ -100,6 +101,7 @@ "AutoModelForVisualQuestionAnswering", "AutoModelForDocumentQuestionAnswering", "AutoModelWithLMHead", + "AutoModelForZeroShotImageClassification", "AutoModelForZeroShotObjectDetection", ] @@ -126,6 +128,7 @@ "TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING", "TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING", "TF_MODEL_FOR_VISION_2_SEQ_MAPPING", + "TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING", "TF_MODEL_MAPPING", "TF_MODEL_WITH_LM_HEAD_MAPPING", "TFAutoModel", @@ -144,6 +147,7 @@ "TFAutoModelForTableQuestionAnswering", "TFAutoModelForTokenClassification", "TFAutoModelForVision2Seq", + "TFAutoModelForZeroShotImageClassification", "TFAutoModelWithLMHead", ] @@ -226,6 +230,7 @@ MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, MODEL_FOR_VISION_2_SEQ_MAPPING, MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING, + MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING, MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING, MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, @@ -258,6 +263,7 @@ AutoModelForVideoClassification, AutoModelForVision2Seq, AutoModelForVisualQuestionAnswering, + AutoModelForZeroShotImageClassification, AutoModelForZeroShotObjectDetection, AutoModelWithLMHead, ) @@ -285,6 +291,7 @@ TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING, TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, TF_MODEL_FOR_VISION_2_SEQ_MAPPING, + TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING, TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING, TFAutoModel, @@ -303,6 +310,7 @@ TFAutoModelForTableQuestionAnswering, TFAutoModelForTokenClassification, TFAutoModelForVision2Seq, + TFAutoModelForZeroShotImageClassification, TFAutoModelWithLMHead, ) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 446ab8ec572d..08fbd4764514 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -920,7 +920,7 @@ ] ) -_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES = OrderedDict( +MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES = OrderedDict( [ # Model for Zero Shot Image Classification mapping ("align", "AlignModel"), @@ -955,6 +955,9 @@ MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING = _LazyAutoMapping( CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES ) +MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING = _LazyAutoMapping( + CONFIG_MAPPING_NAMES, MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES +) MODEL_FOR_IMAGE_SEGMENTATION_MAPPING = _LazyAutoMapping( CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES ) @@ -1142,6 +1145,15 @@ class AutoModelForImageClassification(_BaseAutoModelClass): AutoModelForImageClassification = auto_class_update(AutoModelForImageClassification, head_doc="image classification") +class AutoModelForZeroShotImageClassification(_BaseAutoModelClass): + _model_mapping = MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING + + +AutoModelForZeroShotImageClassification = auto_class_update( + AutoModelForZeroShotImageClassification, head_doc="zero-shot image classification" +) + + class AutoModelForImageSegmentation(_BaseAutoModelClass): _model_mapping = MODEL_FOR_IMAGE_SEGMENTATION_MAPPING diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py index 4d48e6181ebc..caf5ba71dc03 100644 --- a/src/transformers/models/auto/modeling_tf_auto.py +++ b/src/transformers/models/auto/modeling_tf_auto.py @@ -209,6 +209,15 @@ ] ) + +TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES = OrderedDict( + [ + # Model for Zero Shot Image Classification mapping + ("clip", "TFCLIPModel"), + ] +) + + TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES = OrderedDict( [ # Model for Semantic Segmentation mapping @@ -424,6 +433,9 @@ TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING = _LazyAutoMapping( CONFIG_MAPPING_NAMES, TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES ) +TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING = _LazyAutoMapping( + CONFIG_MAPPING_NAMES, TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES +) TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING = _LazyAutoMapping( CONFIG_MAPPING_NAMES, TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES ) @@ -505,6 +517,15 @@ class TFAutoModelForImageClassification(_BaseAutoModelClass): ) +class TFAutoModelForZeroShotImageClassification(_BaseAutoModelClass): + _model_mapping = TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING + + +TFAutoModelForZeroShotImageClassification = auto_class_update( + TFAutoModelForZeroShotImageClassification, head_doc="zero-shot image classification" +) + + class TFAutoModelForSemanticSegmentation(_BaseAutoModelClass): _model_mapping = TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING diff --git a/src/transformers/pipelines/__init__.py b/src/transformers/pipelines/__init__.py index 0f83cb0dea8f..c8c0549a4674 100755 --- a/src/transformers/pipelines/__init__.py +++ b/src/transformers/pipelines/__init__.py @@ -103,6 +103,7 @@ TFAutoModelForTableQuestionAnswering, TFAutoModelForTokenClassification, TFAutoModelForVision2Seq, + TFAutoModelForZeroShotImageClassification, ) if is_torch_available(): @@ -135,6 +136,7 @@ AutoModelForVideoClassification, AutoModelForVision2Seq, AutoModelForVisualQuestionAnswering, + AutoModelForZeroShotImageClassification, AutoModelForZeroShotObjectDetection, ) if TYPE_CHECKING: @@ -290,8 +292,8 @@ }, "zero-shot-image-classification": { "impl": ZeroShotImageClassificationPipeline, - "tf": (TFAutoModel,) if is_tf_available() else (), - "pt": (AutoModel,) if is_torch_available() else (), + "tf": (TFAutoModelForZeroShotImageClassification,) if is_tf_available() else (), + "pt": (AutoModelForZeroShotImageClassification,) if is_torch_available() else (), "default": { "model": { "pt": ("openai/clip-vit-base-patch32", "f4881ba"), diff --git a/src/transformers/pipelines/zero_shot_image_classification.py b/src/transformers/pipelines/zero_shot_image_classification.py index f19a548c85db..8ba07eb018cb 100644 --- a/src/transformers/pipelines/zero_shot_image_classification.py +++ b/src/transformers/pipelines/zero_shot_image_classification.py @@ -18,9 +18,10 @@ from ..image_utils import load_image if is_torch_available(): - pass + from ..models.auto.modeling_auto import MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING if is_tf_available(): + from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING from ..tf_utils import stable_softmax logger = logging.get_logger(__name__) @@ -64,8 +65,11 @@ def __init__(self, **kwargs): super().__init__(**kwargs) requires_backends(self, "vision") - # No specific FOR_XXX available yet - # self.check_model_type(MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING) + self.check_model_type( + TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING + if self.framework == "tf" + else MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING + ) def __call__(self, images: Union[str, List[str], "Image", List["Image"]], **kwargs): """ @@ -137,9 +141,11 @@ def postprocess(self, model_outputs): if self.framework == "pt": probs = logits.softmax(dim=-1).squeeze(-1) scores = probs.tolist() - else: + elif self.framework == "tf": probs = stable_softmax(logits, axis=-1) scores = probs.numpy().tolist() + else: + raise ValueError(f"Unsupported framework: {self.framework}") result = [ {"score": score, "label": candidate_label} diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 85b4010f38c4..62623b4066cb 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -526,6 +526,9 @@ def __init__(self, *args, **kwargs): MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING = None +MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING = None + + MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING = None @@ -738,6 +741,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class AutoModelForZeroShotImageClassification(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class AutoModelForZeroShotObjectDetection(metaclass=DummyObject): _backends = ["torch"] diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index 16d2e6820c16..55eb6599f100 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -316,6 +316,9 @@ def __init__(self, *args, **kwargs): TF_MODEL_FOR_VISION_2_SEQ_MAPPING = None +TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING = None + + TF_MODEL_MAPPING = None @@ -434,6 +437,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) +class TFAutoModelForZeroShotImageClassification(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + class TFAutoModelWithLMHead(metaclass=DummyObject): _backends = ["tf"] diff --git a/src/transformers/utils/fx.py b/src/transformers/utils/fx.py index da9c43b17195..9da544cb45e9 100755 --- a/src/transformers/utils/fx.py +++ b/src/transformers/utils/fx.py @@ -50,6 +50,7 @@ MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES, MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES, + MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES, MODEL_MAPPING_NAMES, ) from ..utils import ENV_VARS_TRUE_VALUES, TORCH_FX_REQUIRED_VERSION, is_torch_fx_available @@ -79,6 +80,7 @@ def _generate_supported_model_class_names( "token-classification": MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES, "masked-image-modeling": MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING_NAMES, "image-classification": MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES, + "zero-shot-image-classification": MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES, "ctc": MODEL_FOR_CTC_MAPPING_NAMES, "audio-classification": MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES, "semantic-segmentation": MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES, diff --git a/utils/update_metadata.py b/utils/update_metadata.py index f95a4575d1e7..8c34bba5d6ae 100644 --- a/utils/update_metadata.py +++ b/utils/update_metadata.py @@ -93,8 +93,8 @@ ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"), ( "zero-shot-image-classification", - "_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES", - "AutoModel", + "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES", + "AutoModelForZeroShotImageClassification", ), ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"), ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"), From 010d2383a3b041f7eab64cfe06cf587f78c0996d Mon Sep 17 00:00:00 2001 From: wangpeng <1204591829@qq.com> Date: Mon, 13 Mar 2023 18:11:31 +0800 Subject: [PATCH 065/277] add new model of MGP-STR (#21418) * add new model of MGP-STR * fix the check failings * remove torch and numpy from mgp_tokenization * remove unused import from modeling_mgp_str * add test_processing_mgp_str * rm test_processing_mgp_str.py * add test_processing_mgp_str * add test_processing_mgp_str * add test_processing_mgp_str * rm test_processing_mgp_str and add softmax outs to model * rm test_processing_mgp_str and add softmax outs to model * rewrite the code of mgp-str according to PR suggestions * rewrite the code of mgp-str according to PR suggestions * add new model of MGP-STR * fix the check failings * remove torch and numpy from mgp_tokenization * remove unused import from modeling_mgp_str * add test_processing_mgp_str * rm test_processing_mgp_str.py * add test_processing_mgp_str * add test_processing_mgp_str * add test_processing_mgp_str * rm test_processing_mgp_str and add softmax outs to model * rewrite the code of mgp-str according to PR suggestions * rewrite the code of mgp-str according to PR suggestions * remove representation_size from MGPSTRConfig * reformat configuration_mgp_str.py * format test_processor_mgp_str.py * add test for tokenizer and complete model/processer test and model file * rm Unnecessary tupple in modeling_mgp_str * reduce hidden_size/layers/label_size in test_model * add integration tests and change MGPSTR to Mgpstr * add test for logit values * reformat test model file --------- Co-authored-by: yue kun --- README.md | 2 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/en/_toctree.yml | 2 + docs/source/en/index.mdx | 2 + docs/source/en/model_doc/mgp-str.mdx | 86 +++ src/transformers/__init__.py | 16 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + .../models/auto/image_processing_auto.py | 1 + src/transformers/models/auto/modeling_auto.py | 1 + .../models/auto/processing_auto.py | 1 + .../models/auto/tokenization_auto.py | 1 + src/transformers/models/mgp_str/__init__.py | 62 +++ .../models/mgp_str/configuration_mgp_str.py | 137 +++++ .../models/mgp_str/modeling_mgp_str.py | 512 ++++++++++++++++++ .../models/mgp_str/processing_mgp_str.py | 228 ++++++++ .../models/mgp_str/tokenization_mgp_str.py | 110 ++++ src/transformers/utils/dummy_pt_objects.py | 24 + tests/models/mgp_str/__init__.py | 0 tests/models/mgp_str/test_modeling_mgp_str.py | 269 +++++++++ .../models/mgp_str/test_processor_mgp_str.py | 211 ++++++++ .../mgp_str/test_tokenization_mgp_str.py | 96 ++++ utils/check_repo.py | 2 + 28 files changed, 1773 insertions(+) create mode 100644 docs/source/en/model_doc/mgp-str.mdx create mode 100644 src/transformers/models/mgp_str/__init__.py create mode 100644 src/transformers/models/mgp_str/configuration_mgp_str.py create mode 100644 src/transformers/models/mgp_str/modeling_mgp_str.py create mode 100644 src/transformers/models/mgp_str/processing_mgp_str.py create mode 100644 src/transformers/models/mgp_str/tokenization_mgp_str.py create mode 100644 tests/models/mgp_str/__init__.py create mode 100644 tests/models/mgp_str/test_modeling_mgp_str.py create mode 100644 tests/models/mgp_str/test_processor_mgp_str.py create mode 100644 tests/models/mgp_str/test_tokenization_mgp_str.py diff --git a/README.md b/README.md index 4110c524c069..4c81448e2ab9 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. @@ -498,3 +499,4 @@ We now have a [paper](https://www.aclweb.org/anthology/2020.emnlp-demos.6/) you pages = "38--45" } ``` + diff --git a/README_es.md b/README_es.md index 728f596eabe2..aa83571ca0ce 100644 --- a/README_es.md +++ b/README_es.md @@ -365,6 +365,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. diff --git a/README_hd.md b/README_hd.md index 0e61392f26ce..ff49b28426d4 100644 --- a/README_hd.md +++ b/README_hd.md @@ -337,6 +337,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (फेसबुक से) साथ में पेपर [एक्स्टेंसिबल बहुभाषी प्रीट्रेनिंग और फाइनट्यूनिंग के साथ बहुभाषी अनुवाद](https://arxiv युकिंग टैंग, चाउ ट्रान, जियान ली, पेंग-जेन चेन, नमन गोयल, विश्रव चौधरी, जियाताओ गु, एंजेला फैन द्वारा .org/abs/2008.00401)। 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA से) कागज के साथ [Megatron-LM: मॉडल का उपयोग करके बहु-अरब पैरामीटर भाषा मॉडल का प्रशिक्षण Parallelism](https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा। 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA से) साथ वाला पेपर [Megatron-LM: ट्रेनिंग मल्टी-बिलियन पैरामीटर लैंग्वेज मॉडल्स यूजिंग मॉडल पैरेललिज़्म] (https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा पोस्ट किया गया। +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research से) Peng Wang, Cheng Da, and Cong Yao. द्वाराअनुसंधान पत्र [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) के साथ जारी किया गया 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (फ्रॉम Studio Ousia) साथ में पेपर [mLUKE: द पावर ऑफ एंटिटी रिप्रेजेंटेशन इन मल्टीलिंगुअल प्रीट्रेन्ड लैंग्वेज मॉडल्स](https://arxiv.org/abs/2110.08151) रयोकन री, इकुया यामाडा, और योशिमासा त्सुरोका द्वारा। 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (सीएमयू/गूगल ब्रेन से) साथ में कागज [मोबाइलबर्ट: संसाधन-सीमित उपकरणों के लिए एक कॉम्पैक्ट टास्क-अज्ञेय बीईआरटी] (https://arxiv.org/abs/2004.02984) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, और Denny Zhou द्वारा पोस्ट किया गया। 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. diff --git a/README_ja.md b/README_ja.md index 4edaeffb83b6..3803a6a2c5ee 100644 --- a/README_ja.md +++ b/README_ja.md @@ -399,6 +399,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook から) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan から公開された研究論文: [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research から) Peng Wang, Cheng Da, and Cong Yao. から公開された研究論文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (Studio Ousia から) Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka から公開された研究論文: [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (CMU/Google Brain から) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou から公開された研究論文: [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (Google Inc. から) Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam から公開された研究論文: [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) diff --git a/README_ko.md b/README_ko.md index ce957d6a925e..18056a12a62e 100644 --- a/README_ko.md +++ b/README_ko.md @@ -314,6 +314,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook 에서) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 의 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 논문과 함께 발표했습니다. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research 에서 제공)은 Peng Wang, Cheng Da, and Cong Yao.의 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592)논문과 함께 발표했습니다. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (Studio Ousia 에서) Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka 의 [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 논문과 함께 발표했습니다. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (CMU/Google Brain 에서) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou 의 [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 논문과 함께 발표했습니다. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (Google Inc. 에서) Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam 의 [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) 논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index ca224409143a..54fb9346b2de 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -338,6 +338,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (来自 Facebook) 伴随论文 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 由 Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 发布。 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (来自 Alibaba Research) 伴随论文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 由 Peng Wang, Cheng Da, and Cong Yao 发布。 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (来自 Studio Ousia) 伴随论文 [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 由 Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka 发布。 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (来自 CMU/Google Brain) 伴随论文 [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 由 Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou 发布。 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (来自 Google Inc.) 伴随论文 [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) 由 Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index e496a6f86c89..08ce35cbfff5 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -350,6 +350,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 773438798682..f164348a94aa 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -580,6 +580,8 @@ title: LiLT - local: model_doc/lxmert title: LXMERT + - local: model_doc/mgp-str + title: MGP-STR - local: model_doc/oneformer title: OneFormer - local: model_doc/owlvit diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 5a547ba8ef56..36964e924a81 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -151,6 +151,7 @@ The documentation is organized into five sections: 1. **[mBART-50](model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. +1. **[MGP-STR](model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. @@ -340,6 +341,7 @@ Flax), PyTorch, and/or TensorFlow. | MaskFormerSwin | ❌ | ❌ | ❌ | ❌ | ❌ | | mBART | ✅ | ✅ | ✅ | ✅ | ✅ | | Megatron-BERT | ❌ | ❌ | ✅ | ❌ | ❌ | +| MGP-STR | ✅ | ❌ | ✅ | ❌ | ❌ | | MobileBERT | ✅ | ✅ | ✅ | ✅ | ❌ | | MobileNetV1 | ❌ | ❌ | ✅ | ❌ | ❌ | | MobileNetV2 | ❌ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/model_doc/mgp-str.mdx b/docs/source/en/model_doc/mgp-str.mdx new file mode 100644 index 000000000000..fc3d9f41526d --- /dev/null +++ b/docs/source/en/model_doc/mgp-str.mdx @@ -0,0 +1,86 @@ + + +# MGP-STR + +## Overview + +The MGP-STR model was proposed in [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. MGP-STR is a conceptually **simple** yet **powerful** vision Scene Text Recognition (STR) model, which is built upon the [Vision Transformer (ViT)](vit). To integrate linguistic knowledge, Multi-Granularity Prediction (MGP) strategy is proposed to inject information from the language modality into the model in an implicit way. + +The abstract from the paper is the following: + +*Scene text recognition (STR) has been an active research topic in computer vision for years. To tackle this challenging problem, numerous innovative methods have been successively proposed and incorporating linguistic knowledge into STR models has recently become a prominent trend. In this work, we first draw inspiration from the recent progress in Vision Transformer (ViT) to construct a conceptually simple yet powerful vision STR model, which is built upon ViT and outperforms previous state-of-the-art models for scene text recognition, including both pure vision models and language-augmented methods. To integrate linguistic knowledge, we further propose a Multi-Granularity Prediction strategy to inject information from the language modality into the model in an implicit way, i.e. , subword representations (BPE and WordPiece) widely-used in NLP are introduced into the output space, in addition to the conventional character level representation, while no independent language model (LM) is adopted. The resultant algorithm (termed MGP-STR) is able to push the performance envelop of STR to an even higher level. Specifically, it achieves an average recognition accuracy of 93.35% on standard benchmarks.* + + + + MGP-STR architecture. Taken from the original paper. + +Tips: + +- MGP-STR is trained on two synthetic datasets [MJSynth]((http://www.robots.ox.ac.uk/~vgg/data/text/)) (MJ) and SynthText(http://www.robots.ox.ac.uk/~vgg/data/scenetext/) (ST) without fine-tuning on other datasets. It achieves state-of-the-art results on six standard Latin scene text benchmarks, including 3 regular text datasets (IC13, SVT, IIIT) and 3 irregular ones (IC15, SVTP, CUTE). +- This model was contributed by [yuekun](https://huggingface.co/yuekun). The original code can be found [here](https://github.com/AlibabaResearch/AdvancedLiterateMachinery/tree/main/OCR/MGP-STR). + +## Inference + +[`MgpstrModel`] accepts images as input and generates three types of predictions, which represent textual information at different granularities. +The three types of predictions are fused to give the final prediction result. + +The [`ViTImageProcessor`] class is responsible for preprocessing the input image and +[`MgpstrTokenizer`] decodes the generated character tokens to the target string. The +[`MgpstrProcessor`] wraps [`ViTImageProcessor`] and [`MgpstrTokenizer`] +into a single instance to both extract the input features and decode the predicted token ids. + +- Step-by-step Optical Character Recognition (OCR) + +``` py +>>> from transformers import MgpstrProcessor, MgpstrForSceneTextRecognition +>>> import requests +>>> from PIL import Image + +>>> processor = MgpstrProcessor.from_pretrained('alibaba-damo/mgp-str-base') +>>> model = MgpstrForSceneTextRecognition.from_pretrained('alibaba-damo/mgp-str-base') + +>>> # load image from the IIIT-5k dataset +>>> url = "https://i.postimg.cc/ZKwLg2Gw/367-14.png" +>>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB") + +>>> pixel_values = processor(image, return_tensors="pt").pixel_values +>>> outputs = model(pixel_values) + +>>> generated_text = processor.batch_decode(outputs.logits)['generated_text'] +``` + +## MgpstrConfig + +[[autodoc]] MgpstrConfig + +## MgpstrTokenizer + +[[autodoc]] MgpstrTokenizer + - save_vocabulary + +## MgpstrProcessor + +[[autodoc]] MgpstrProcessor + - __call__ + - batch_decode + +## MgpstrModel + +[[autodoc]] MgpstrModel + - forward + +## MgpstrForSceneTextRecognition + +[[autodoc]] MgpstrForSceneTextRecognition + - forward diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index f32f3c417e94..8d41feb8abd2 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -370,6 +370,7 @@ "models.mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig", "MCTCTProcessor"], "models.megatron_bert": ["MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegatronBertConfig"], "models.megatron_gpt2": [], + "models.mgp_str": ["MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP", "MgpstrConfig", "MgpstrProcessor", "MgpstrTokenizer"], "models.mluke": [], "models.mmbt": ["MMBTConfig"], "models.mobilebert": ["MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MobileBertConfig", "MobileBertTokenizer"], @@ -1904,6 +1905,14 @@ "MegatronBertPreTrainedModel", ] ) + _import_structure["models.mgp_str"].extend( + [ + "MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST", + "MgpstrForSceneTextRecognition", + "MgpstrModel", + "MgpstrPreTrainedModel", + ] + ) _import_structure["models.mmbt"].extend(["MMBTForClassification", "MMBTModel", "ModalEmbeddings"]) _import_structure["models.mobilebert"].extend( [ @@ -3963,6 +3972,7 @@ from .models.mbart import MBartConfig from .models.mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig, MCTCTProcessor from .models.megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig + from .models.mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig, MgpstrProcessor, MgpstrTokenizer from .models.mmbt import MMBTConfig from .models.mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig, MobileBertTokenizer from .models.mobilenet_v1 import MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileNetV1Config @@ -5250,6 +5260,12 @@ MegatronBertModel, MegatronBertPreTrainedModel, ) + from .models.mgp_str import ( + MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST, + MgpstrForSceneTextRecognition, + MgpstrModel, + MgpstrPreTrainedModel, + ) from .models.mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings from .models.mobilebert import ( MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 35017e3e0320..29d023fc6a5d 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -116,6 +116,7 @@ mctct, megatron_bert, megatron_gpt2, + mgp_str, mluke, mmbt, mobilebert, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index ccd516c2d15b..1ab0b62c793a 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -121,6 +121,7 @@ ("mbart", "MBartConfig"), ("mctct", "MCTCTConfig"), ("megatron-bert", "MegatronBertConfig"), + ("mgp-str", "MgpstrConfig"), ("mobilebert", "MobileBertConfig"), ("mobilenet_v1", "MobileNetV1Config"), ("mobilenet_v2", "MobileNetV2Config"), @@ -294,6 +295,7 @@ ("mbart", "MBART_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mctct", "MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("megatron-bert", "MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("mgp-str", "MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mobilenet_v1", "MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mobilenet_v2", "MOBILENET_V2_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mobilevit", "MOBILEVIT_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -476,6 +478,7 @@ ("mctct", "M-CTC-T"), ("megatron-bert", "Megatron-BERT"), ("megatron_gpt2", "Megatron-GPT2"), + ("mgp-str", "MGP-STR"), ("mluke", "mLUKE"), ("mobilebert", "MobileBERT"), ("mobilenet_v1", "MobileNetV1"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 8b45c4d65140..6274c88f7d31 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -69,6 +69,7 @@ ("levit", "LevitImageProcessor"), ("mask2former", "Mask2FormerImageProcessor"), ("maskformer", "MaskFormerImageProcessor"), + ("mgp-str", "ViTImageProcessor"), ("mobilenet_v1", "MobileNetV1ImageProcessor"), ("mobilenet_v2", "MobileNetV2ImageProcessor"), ("mobilenet_v2", "MobileNetV2ImageProcessor"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 08fbd4764514..f871de8e55f2 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -119,6 +119,7 @@ ("mbart", "MBartModel"), ("mctct", "MCTCTModel"), ("megatron-bert", "MegatronBertModel"), + ("mgp-str", "MgpstrForSceneTextRecognition"), ("mobilebert", "MobileBertModel"), ("mobilenet_v1", "MobileNetV1Model"), ("mobilenet_v2", "MobileNetV2Model"), diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index 197cfe8e79c5..29726fde4f08 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -57,6 +57,7 @@ ("layoutlmv2", "LayoutLMv2Processor"), ("layoutlmv3", "LayoutLMv3Processor"), ("markuplm", "MarkupLMProcessor"), + ("mgp-str", "MgpstrProcessor"), ("oneformer", "OneFormerProcessor"), ("owlvit", "OwlViTProcessor"), ("sew", "Wav2Vec2Processor"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index f5035ab33180..3f81c5e1f935 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -194,6 +194,7 @@ ), ), ("megatron-bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), + ("mgp-str", ("MgpstrTokenizer", None)), ("mluke", ("MLukeTokenizer" if is_sentencepiece_available() else None, None)), ("mobilebert", ("MobileBertTokenizer", "MobileBertTokenizerFast" if is_tokenizers_available() else None)), ("mpnet", ("MPNetTokenizer", "MPNetTokenizerFast" if is_tokenizers_available() else None)), diff --git a/src/transformers/models/mgp_str/__init__.py b/src/transformers/models/mgp_str/__init__.py new file mode 100644 index 000000000000..01c0ab7fd9f0 --- /dev/null +++ b/src/transformers/models/mgp_str/__init__.py @@ -0,0 +1,62 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available + + +_import_structure = { + "configuration_mgp_str": ["MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP", "MgpstrConfig"], + "processing_mgp_str": ["MgpstrProcessor"], + "tokenization_mgp_str": ["MgpstrTokenizer"], +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_mgp_str"] = [ + "MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST", + "MgpstrModel", + "MgpstrPreTrainedModel", + "MgpstrForSceneTextRecognition", + ] + +if TYPE_CHECKING: + from .configuration_mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig + from .processing_mgp_str.py import MgpstrProcessor + from .tokenization_mgp_str import MgpstrTokenizer + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_mgp_str import ( + MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST, + MgpstrForSceneTextRecognition, + MgpstrModel, + MgpstrPreTrainedModel, + ) +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/mgp_str/configuration_mgp_str.py b/src/transformers/models/mgp_str/configuration_mgp_str.py new file mode 100644 index 000000000000..e77248cd6449 --- /dev/null +++ b/src/transformers/models/mgp_str/configuration_mgp_str.py @@ -0,0 +1,137 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MGP-STR model configuration""" + +from ...configuration_utils import PretrainedConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "alibaba-damo/mgp-str-base": "https://huggingface.co/alibaba-damo/mgp-str-base/resolve/main/config.json", +} + + +class MgpstrConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of an [`MgpstrModel`]. It is used to instantiate an + MGP-STR model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the MGP-STR + [alibaba-damo/mgp-str-base](https://huggingface.co/alibaba-damo/mgp-str-base) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + image_size (`List[int]`, *optional*, defaults to `[32, 128]`): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to 4): + The size (resolution) of each patch. + num_channels (`int`, *optional*, defaults to 3): + The number of input channels. + max_token_length (`int`, *optional*, defaults to 27): + The max number of output tokens. + num_character_labels (`int`, *optional*, defaults to 38): + The number of classes for character head . + num_bpe_labels (`int`, *optional*, defaults to 50257): + The number of classes for bpe head . + num_wordpiece_labels (`int`, *optional*, defaults to 30522): + The number of classes for wordpiece head . + hidden_size (`int`, *optional*, defaults to 768): + The embedding dimension. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + mlp_ratio (`float`, *optional*, defaults to 4.0): + The ratio of mlp hidden dim to embedding dim. + qkv_bias (`bool`, *optional*, defaults to `True`): + Whether to add a bias to the queries, keys and values. + distilled (`bool`, *optional*, defaults to `False`): + Model includes a distillation token and head as in DeiT models. + layer_norm_eps (`float`, *optional*, defaults to 1e-5): + The epsilon used by the layer normalization layers. + drop_rate (`float`, *optional*, defaults to 0.0): + The dropout probability for all fully connected layers in the embeddings, encoder. + attn_drop_rate (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + drop_path_rate (`float`, *optional*, defaults to 0.0): + The stochastic depth rate. + output_a3_attentions (`bool`, *optional*, defaults to `False`): + Whether or not the model should returns A^3 module attentions. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + + Example: + + ```python + >>> from transformers import MgpstrConfig, MgpstrForSceneTextRecognition + + >>> # Initializing a Mgpstr mgp-str-base style configuration + >>> configuration = MgpstrConfig() + + >>> # Initializing a model (with random weights) from the mgp-str-base style configuration + >>> model = MgpstrForSceneTextRecognition(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "mgp-str" + + def __init__( + self, + image_size=[32, 128], + patch_size=4, + num_channels=3, + max_token_length=27, + num_character_labels=38, + num_bpe_labels=50257, + num_wordpiece_labels=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + mlp_ratio=4.0, + qkv_bias=True, + distilled=False, + layer_norm_eps=1e-5, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + output_a3_attentions=False, + initializer_range=0.02, + **kwargs, + ): + super().__init__(**kwargs) + + self.image_size = image_size + self.patch_size = patch_size + self.num_channels = num_channels + self.max_token_length = max_token_length + self.num_character_labels = num_character_labels + self.num_bpe_labels = num_bpe_labels + self.num_wordpiece_labels = num_wordpiece_labels + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.mlp_ratio = mlp_ratio + self.distilled = distilled + self.layer_norm_eps = layer_norm_eps + self.drop_rate = drop_rate + self.qkv_bias = qkv_bias + self.attn_drop_rate = attn_drop_rate + self.drop_path_rate = drop_path_rate + self.output_a3_attentions = output_a3_attentions + self.initializer_range = initializer_range diff --git a/src/transformers/models/mgp_str/modeling_mgp_str.py b/src/transformers/models/mgp_str/modeling_mgp_str.py new file mode 100644 index 000000000000..35ed55f5f578 --- /dev/null +++ b/src/transformers/models/mgp_str/modeling_mgp_str.py @@ -0,0 +1,512 @@ +# coding=utf-8 +# Copyright 2023 Alibaba Research and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch MGP-STR model.""" + +import collections.abc +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn + +from ...modeling_outputs import BaseModelOutput +from ...modeling_utils import PreTrainedModel +from ...utils import ( + ModelOutput, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_mgp_str import MgpstrConfig + + +logger = logging.get_logger(__name__) + +# General docstring +_CONFIG_FOR_DOC = "MgpstrConfig" +_TOKENIZER_FOR_DOC = "MgpstrTokenizer" + +# Base docstring +_CHECKPOINT_FOR_DOC = "alibaba-damo/mgp-str-base" + +MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "alibaba-damo/mgp-str-base", + # See all MGP-STR models at https://huggingface.co/models?filter=mgp-str +] + + +# Copied from transformers.models.beit.modeling_beit.drop_path +def drop_path(input, drop_prob: float = 0.0, training: bool = False): + """ + Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks, + however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the + layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the + argument. + """ + if drop_prob == 0.0 or not training: + return input + keep_prob = 1 - drop_prob + shape = (input.shape[0],) + (1,) * (input.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=input.dtype, device=input.device) + random_tensor.floor_() # binarize + output = input.div(keep_prob) * random_tensor + return output + + +# Copied from transformers.models.beit.modeling_beit.BeitDropPath with Beit->Mgpstr +class MgpstrDropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob: Optional[float] = None) -> None: + super().__init__() + self.drop_prob = drop_prob + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return drop_path(hidden_states, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return "p={}".format(self.drop_prob) + + +@dataclass +class MgpstrModelOutput(ModelOutput): + """ + Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. + + Args: + logits (`tuple(torch.FloatTensor)` of shape `(batch_size, config.num_character_labels)`): + Tuple of `torch.FloatTensor` (one for the output of character of shape `(batch_size, + config.max_token_length, config.num_character_labels)`, + one for the output of bpe of shape `(batch_size, + config.max_token_length, config.num_bpe_labels)`, + one for the output of wordpiece of shape `(batch_size, + config.max_token_length, config.num_wordpiece_labels)`) . + + Classification scores (before SoftMax) of character, bpe and wordpiece. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, config.max_token_length, + sequence_length, sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + a3_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_a3_attentions=True` is passed or when `config.output_a3_attentions=True`): + Tuple of `torch.FloatTensor` (one for the attention of character, + one for the attention of bpe`, + one + for the attention of wordpiece) of shape `(batch_size, config.max_token_length, sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + logits: Tuple[torch.FloatTensor] = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + a3_attentions: Optional[Tuple[torch.FloatTensor]] = None + + +class MgpstrEmbeddings(nn.Module): + """2D Image to Patch Embedding""" + + def __init__(self, config: MgpstrConfig): + super().__init__() + image_size = ( + config.image_size + if isinstance(config.image_size, collections.abc.Iterable) + else (config.image_size, config.image_size) + ) + patch_size = ( + config.patch_size + if isinstance(config.patch_size, collections.abc.Iterable) + else (config.patch_size, config.patch_size) + ) + self.image_size = image_size + self.patch_size = patch_size + self.grid_size = (image_size[0] // patch_size[0], image_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.num_tokens = 2 if config.distilled else 1 + + self.proj = nn.Conv2d(config.num_channels, config.hidden_size, kernel_size=patch_size, stride=patch_size) + + self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) + + self.pos_embed = nn.Parameter(torch.zeros(1, self.num_patches + self.num_tokens, config.hidden_size)) + self.pos_drop = nn.Dropout(p=config.drop_rate) + + def forward(self, pixel_values): + batch_size, channel, height, width = pixel_values.shape + if height != self.image_size[0] or width != self.image_size[1]: + raise ValueError( + f"Input image size ({height}*{width}) doesn't match model ({self.image_size[0]}*{self.image_size[1]})." + ) + + patch_embeddings = self.proj(pixel_values) + patch_embeddings = patch_embeddings.flatten(2).transpose(1, 2) # BCHW -> BNC + + cls_tokens = self.cls_token.expand(batch_size, -1, -1) + embedding_output = torch.cat((cls_tokens, patch_embeddings), dim=1) + embedding_output = embedding_output + self.pos_embed + embedding_output = self.pos_drop(embedding_output) + + return embedding_output + + +class MgpstrMlp(nn.Module): + """MLP as used in Vision Transformer, MLP-Mixer and related networks""" + + def __init__(self, config: MgpstrConfig, hidden_features): + super().__init__() + hidden_features = hidden_features or config.hidden_size + self.fc1 = nn.Linear(config.hidden_size, hidden_features) + self.act = nn.GELU() + self.fc2 = nn.Linear(hidden_features, config.hidden_size) + self.drop = nn.Dropout(config.drop_rate) + + def forward(self, hidden_states): + hidden_states = self.fc1(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.drop(hidden_states) + hidden_states = self.fc2(hidden_states) + hidden_states = self.drop(hidden_states) + return hidden_states + + +class MgpstrAttention(nn.Module): + def __init__(self, config: MgpstrConfig): + super().__init__() + self.num_heads = config.num_attention_heads + head_dim = config.hidden_size // config.num_attention_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(config.hidden_size, config.hidden_size * 3, bias=config.qkv_bias) + self.attn_drop = nn.Dropout(config.attn_drop_rate) + self.proj = nn.Linear(config.hidden_size, config.hidden_size) + self.proj_drop = nn.Dropout(config.drop_rate) + + def forward(self, hidden_states): + batch_size, num, channel = hidden_states.shape + qkv = ( + self.qkv(hidden_states) + .reshape(batch_size, num, 3, self.num_heads, channel // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + query, key, value = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + attention_probs = (query @ key.transpose(-2, -1)) * self.scale + attention_probs = attention_probs.softmax(dim=-1) + attention_probs = self.attn_drop(attention_probs) + + context_layer = (attention_probs @ value).transpose(1, 2).reshape(batch_size, num, channel) + context_layer = self.proj(context_layer) + context_layer = self.proj_drop(context_layer) + return (context_layer, attention_probs) + + +class MgpstrLayer(nn.Module): + def __init__(self, config: MgpstrConfig, drop_path=None): + super().__init__() + self.norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attn = MgpstrAttention(config) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = MgpstrDropPath(drop_path) if drop_path is not None else nn.Identity() + self.norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + mlp_hidden_dim = int(config.hidden_size * config.mlp_ratio) + self.mlp = MgpstrMlp(config, mlp_hidden_dim) + + def forward(self, hidden_states): + self_attention_outputs = self.attn(self.norm1(hidden_states)) + attention_output = self_attention_outputs[0] + outputs = self_attention_outputs[1] + + # first residual connection + hidden_states = self.drop_path(attention_output) + hidden_states + + # second residual connection is done here + layer_output = hidden_states + self.drop_path(self.mlp(self.norm2(hidden_states))) + + outputs = (layer_output, outputs) + return outputs + + +class MgpstrEncoder(nn.Module): + def __init__(self, config: MgpstrConfig): + super().__init__() + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers)] + + self.blocks = nn.Sequential( + *[MgpstrLayer(config=config, drop_path=dpr[i]) for i in range(config.num_hidden_layers)] + ) + + def forward(self, hidden_states, output_attentions=False, output_hidden_states=False, return_dict=True): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + + for _, blk in enumerate(self.blocks): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = blk(hidden_states) + hidden_states = layer_outputs[0] + + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + +class MgpstrA3Module(nn.Module): + def __init__(self, config: MgpstrConfig): + super().__init__() + self.token_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.tokenLearner = nn.Sequential( + nn.Conv2d(config.hidden_size, config.hidden_size, kernel_size=(1, 1), stride=1, groups=8, bias=False), + nn.Conv2d(config.hidden_size, config.max_token_length, kernel_size=(1, 1), stride=1, bias=False), + ) + self.feat = nn.Conv2d( + config.hidden_size, config.hidden_size, kernel_size=(1, 1), stride=1, groups=8, bias=False + ) + self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.token_norm(hidden_states) + hidden_states = hidden_states.transpose(1, 2).unsqueeze(-1) + selected = self.tokenLearner(hidden_states) + selected = selected.flatten(2) + attentions = F.softmax(selected, dim=-1) + + feat = self.feat(hidden_states) + feat = feat.flatten(2).transpose(1, 2) + feat = torch.einsum("...si,...id->...sd", attentions, feat) + a3_out = self.norm(feat) + + return (a3_out, attentions) + + +class MgpstrPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = MgpstrConfig + base_model_prefix = "mgp_str" + + def _init_weights(self, module: Union[nn.Linear, nn.Conv2d, nn.LayerNorm]) -> None: + """Initialize the weights""" + if isinstance(module, MgpstrEmbeddings): + nn.init.trunc_normal_(module.pos_embed, mean=0.0, std=self.config.initializer_range) + nn.init.trunc_normal_(module.cls_token, mean=0.0, std=self.config.initializer_range) + elif isinstance(module, (nn.Linear, nn.Conv2d)): + module.weight.data = nn.init.trunc_normal_(module.weight.data, mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module: MgpstrEncoder, value: bool = False) -> None: + if isinstance(module, MgpstrEncoder): + module.gradient_checkpointing = value + + +MGP_STR_START_DOCSTRING = r""" + This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it + as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and + behavior. + + Parameters: + config ([`MgpstrConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +MGP_STR_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`] + for details. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare MGP-STR Model transformer outputting raw hidden-states without any specific head on top.", + MGP_STR_START_DOCSTRING, +) +class MgpstrModel(MgpstrPreTrainedModel): + def __init__(self, config: MgpstrConfig): + super().__init__(config) + self.config = config + self.embeddings = MgpstrEmbeddings(config) + self.encoder = MgpstrEncoder(config) + + def get_input_embeddings(self) -> nn.Module: + return self.embeddings.proj + + @add_start_docstrings_to_model_forward(MGP_STR_INPUTS_DOCSTRING) + def forward(self, pixel_values, output_attentions=None, output_hidden_states=None, return_dict=None): + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + embedding_output = self.embeddings(pixel_values) + + encoder_outputs = self.encoder( + embedding_output, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return encoder_outputs + return BaseModelOutput( + last_hidden_state=encoder_outputs.last_hidden_state, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +@add_start_docstrings( + """ + MGP-STR Model transformer with three classification heads on top (three A^3 modules and three linear layer on top + of the transformer encoder output) for scene text recognition (STR) . + """, + MGP_STR_START_DOCSTRING, +) +class MgpstrForSceneTextRecognition(MgpstrPreTrainedModel): + config_class = MgpstrConfig + main_input_name = "pixel_values" + + def __init__(self, config: MgpstrConfig) -> None: + super().__init__(config) + + self.num_labels = config.num_labels + self.mgp_str = MgpstrModel(config) + + self.char_a3_module = MgpstrA3Module(config) + self.bpe_a3_module = MgpstrA3Module(config) + self.wp_a3_module = MgpstrA3Module(config) + + self.char_head = nn.Linear(config.hidden_size, config.num_character_labels) + self.bpe_head = nn.Linear(config.hidden_size, config.num_bpe_labels) + self.wp_head = nn.Linear(config.hidden_size, config.num_wordpiece_labels) + + @add_start_docstrings_to_model_forward(MGP_STR_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=MgpstrModelOutput, config_class=MgpstrConfig) + def forward( + self, + pixel_values, + output_attentions=None, + output_a3_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + output_a3_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of a3 modules. See `a3_attentions` under returned tensors + for more detail. + + Returns: + + Example: + + ```python + >>> from transformers import ( + ... MgpstrProcessor, + ... MgpstrForSceneTextRecognition, + ... ) + >>> import requests + >>> from PIL import Image + + >>> # load image from the IIIT-5k dataset + >>> url = "https://i.postimg.cc/ZKwLg2Gw/367-14.png" + >>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB") + + >>> processor = MgpstrProcessor.from_pretrained("alibaba-damo/mgp-str-base") + >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values + + >>> model = MgpstrForSceneTextRecognition.from_pretrained("alibaba-damo/mgp-str-base") + + >>> # inference + >>> outputs = model(pixel_values) + >>> out_strs = processor.batch_decode(outputs.logits) + >>> out_strs["generated_text"] + '["ticket"]' + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + mgp_outputs = self.mgp_str( + pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = mgp_outputs[0] + + char_a3_out, char_attention = self.char_a3_module(sequence_output) + bpe_a3_out, bpe_attention = self.bpe_a3_module(sequence_output) + wp_a3_out, wp_attention = self.wp_a3_module(sequence_output) + + char_logits = self.char_head(char_a3_out) + bpe_logits = self.bpe_head(bpe_a3_out) + wp_logits = self.wp_head(wp_a3_out) + + all_a3_attentions = (char_attention, bpe_attention, wp_attention) if output_a3_attentions else None + all_logits = (char_logits, bpe_logits, wp_logits) + + if not return_dict: + outputs = (all_logits, all_a3_attentions) + mgp_outputs[1:] + return tuple(output for output in outputs if output is not None) + return MgpstrModelOutput( + logits=all_logits, + hidden_states=mgp_outputs.hidden_states, + attentions=mgp_outputs.attentions, + a3_attentions=all_a3_attentions, + ) diff --git a/src/transformers/models/mgp_str/processing_mgp_str.py b/src/transformers/models/mgp_str/processing_mgp_str.py new file mode 100644 index 000000000000..76143deead64 --- /dev/null +++ b/src/transformers/models/mgp_str/processing_mgp_str.py @@ -0,0 +1,228 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Processor class for MGP-STR.""" + +import warnings + +from transformers import AutoTokenizer +from transformers.utils import is_torch_available +from transformers.utils.generic import ExplicitEnum + +from ...processing_utils import ProcessorMixin + + +if is_torch_available(): + import torch + + +class DecodeType(ExplicitEnum): + CHARACTER = "char" + BPE = "bpe" + WORDPIECE = "wp" + + +SUPPORTED_ANNOTATION_FORMATS = (DecodeType.CHARACTER, DecodeType.BPE, DecodeType.WORDPIECE) + + +class MgpstrProcessor(ProcessorMixin): + r""" + Constructs a MGP-STR processor which wraps an image processor and MGP-STR tokenizers into a single + + [`MgpstrProcessor`] offers all the functionalities of `ViTImageProcessor`] and [`MgpstrTokenizer`]. See the + [`~MgpstrProcessor.__call__`] and [`~MgpstrProcessor.batch_decode`] for more information. + + Args: + image_processor (`ViTImageProcessor`): + An instance of `ViTImageProcessor`. The image processor is a required input. + tokenizer ([`MgpstrTokenizer`]): + The tokenizer is a required input. + """ + attributes = ["image_processor", "char_tokenizer"] + image_processor_class = "ViTImageProcessor" + char_tokenizer_class = "MgpstrTokenizer" + + def __init__(self, image_processor=None, tokenizer=None, **kwargs): + if "feature_extractor" in kwargs: + warnings.warn( + "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`" + " instead.", + FutureWarning, + ) + feature_extractor = kwargs.pop("feature_extractor") + + image_processor = image_processor if image_processor is not None else feature_extractor + if image_processor is None: + raise ValueError("You need to specify an `image_processor`.") + if tokenizer is None: + raise ValueError("You need to specify a `tokenizer`.") + + self.char_tokenizer = tokenizer + self.bpe_tokenizer = AutoTokenizer.from_pretrained("gpt2") + self.wp_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") + + super().__init__(image_processor, tokenizer) + + def __call__(self, text=None, images=None, return_tensors=None, **kwargs): + """ + When used in normal mode, this method forwards all its arguments to ViTImageProcessor's + [`~ViTImageProcessor.__call__`] and returns its output. This method also forwards the `text` and `kwargs` + arguments to MgpstrTokenizer's [`~MgpstrTokenizer.__call__`] if `text` is not `None` to encode the text. Please + refer to the doctsring of the above methods for more information. + """ + if images is None and text is None: + raise ValueError("You need to specify either an `images` or `text` input to process.") + + if images is not None: + inputs = self.image_processor(images, return_tensors=return_tensors, **kwargs) + if text is not None: + encodings = self.char_tokenizer(text, return_tensors=return_tensors, **kwargs) + + if text is None: + return inputs + elif images is None: + return encodings + else: + inputs["labels"] = encodings["input_ids"] + return inputs + + def batch_decode(self, sequences): + """ + Convert a list of lists of token ids into a list of strings by calling decode. + + Args: + sequences (`torch.Tensor`): + List of tokenized input ids. + + Returns: + `Dict[str, any]`: Dictionary of all the outputs of the decoded results. + generated_text (`List[str]`): The final results after fusion of char, bpe, and wp. scores + (`List[float]`): The final scores after fusion of char, bpe, and wp. char_preds (`List[str]`): The list + of character decoded sentences. bpe_preds (`List[str]`): The list of bpe decoded sentences. wp_preds + (`List[str]`): The list of wp decoded sentences. + + This method forwards all its arguments to PreTrainedTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please + refer to the docstring of this method for more information. + """ + char_preds, bpe_preds, wp_preds = sequences + batch_size = char_preds.size(0) + + char_strs, char_scores = self._decode_helper(char_preds, "char") + bpe_strs, bpe_scores = self._decode_helper(bpe_preds, "bpe") + wp_strs, wp_scores = self._decode_helper(wp_preds, "wp") + + final_strs = [] + final_scores = [] + for i in range(batch_size): + scores = [char_scores[i], bpe_scores[i], wp_scores[i]] + strs = [char_strs[i], bpe_strs[i], wp_strs[i]] + max_score_index = scores.index(max(scores)) + final_strs.append(strs[max_score_index]) + final_scores.append(scores[max_score_index]) + + out = {} + out["generated_text"] = final_strs + out["scores"] = final_scores + out["char_preds"] = char_strs + out["bpe_preds"] = bpe_strs + out["wp_preds"] = wp_strs + return out + + def _decode_helper(self, pred_logits, format): + """ + Convert a list of lists of bpe token ids into a list of strings by calling bpe tokenizer. + + Args: + pred_logits (`torch.Tensor`): + List of model prediction logits. + format (`Union[DecoderType, str]`): + Type of model prediction. Must be one of ['char', 'bpe', 'wp']. + Returns: + `tuple`: + dec_strs(`str`): The decode strings of model prediction. conf_scores(`List[float]`): The confidence + score of model prediction. + """ + if format == DecodeType.CHARACTER: + decoder = self.char_decode + eos_token = 1 + eos_str = "[s]" + elif format == DecodeType.BPE: + decoder = self.bpe_decode + eos_token = 2 + eos_str = "#" + elif format == DecodeType.WORDPIECE: + decoder = self.wp_decode + eos_token = 102 + eos_str = "[SEP]" + else: + raise ValueError(f"Format {format} is not supported.") + + dec_strs, conf_scores = [], [] + batch_size = pred_logits.size(0) + batch_max_length = pred_logits.size(1) + _, preds_index = pred_logits.topk(1, dim=-1, largest=True, sorted=True) + preds_index = preds_index.view(-1, batch_max_length)[:, 1:] + preds_str = decoder(preds_index) + preds_max_prob, _ = torch.nn.functional.softmax(pred_logits, dim=2).max(dim=2) + preds_max_prob = preds_max_prob[:, 1:] + + for index in range(batch_size): + pred_eos = preds_str[index].find(eos_str) + pred = preds_str[index][:pred_eos] + pred_index = preds_index[index].cpu().tolist() + pred_eos_index = pred_index.index(eos_token) if eos_token in pred_index else -1 + pred_max_prob = preds_max_prob[index][: pred_eos_index + 1] + confidence_score = pred_max_prob.cumprod(dim=0)[-1] if pred_max_prob.nelement() != 0 else 0.0 + dec_strs.append(pred) + conf_scores.append(confidence_score) + + return dec_strs, conf_scores + + def char_decode(self, sequences): + """ + Convert a list of lists of char token ids into a list of strings by calling char tokenizer. + + Args: + sequences (`torch.Tensor`): + List of tokenized input ids. + Returns: + `List[str]`: The list of char decoded sentences. + """ + decode_strs = [seq.replace(" ", "") for seq in self.char_tokenizer.batch_decode(sequences)] + return decode_strs + + def bpe_decode(self, sequences): + """ + Convert a list of lists of bpe token ids into a list of strings by calling bpe tokenizer. + + Args: + sequences (`torch.Tensor`): + List of tokenized input ids. + Returns: + `List[str]`: The list of bpe decoded sentences. + """ + return self.bpe_tokenizer.batch_decode(sequences) + + def wp_decode(self, sequences): + """ + Convert a list of lists of word piece token ids into a list of strings by calling word piece tokenizer. + + Args: + sequences (`torch.Tensor`): + List of tokenized input ids. + Returns: + `List[str]`: The list of wp decoded sentences. + """ + decode_strs = [seq.replace(" ", "") for seq in self.wp_tokenizer.batch_decode(sequences)] + return decode_strs diff --git a/src/transformers/models/mgp_str/tokenization_mgp_str.py b/src/transformers/models/mgp_str/tokenization_mgp_str.py new file mode 100644 index 000000000000..9d4fddcc7e83 --- /dev/null +++ b/src/transformers/models/mgp_str/tokenization_mgp_str.py @@ -0,0 +1,110 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes for MGT-STR CHAR.""" + +import json +import os +from typing import Optional, Tuple + +from ...tokenization_utils import PreTrainedTokenizer +from ...utils import logging + + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "vocab.json"} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "mgp-str": "https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"mgp-str": 27} + + +class MgpstrTokenizer(PreTrainedTokenizer): + """ + Construct a MGP-STR char tokenizer. + + This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to + this superclass for more information regarding those methods. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + unk_token (`str`, *optional*, defaults to `"[GO]"`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + bos_token (`str`, *optional*, defaults to `"[GO]"`): + The beginning of sequence token. + eos_token (`str`, *optional*, defaults to `"[s]"`): + The end of sequence token. + pad_token (`str` or `tokenizers.AddedToken`, *optional*, , defaults to `"[GO]"`): + A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by + attention mechanisms or loss computation. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, unk_token="[GO]", bos_token="[GO]", eos_token="[s]", pad_token="[GO]", **kwargs): + super().__init__( + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + **kwargs, + ) + + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.vocab = json.load(vocab_handle) + self.decoder = {v: k for k, v in self.vocab.items()} + + @property + def vocab_size(self): + return len(self.vocab) + + def get_vocab(self): + return dict(self.vocab, **self.added_tokens_encoder) + + def _tokenize(self, text): + """Tokenize a string.""" + char_tokens = [] + for s in text: + char_tokens.extend(s) + return char_tokens + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.vocab.get(token, self.vocab.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.decoder.get(index) + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + + with open(vocab_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.vocab, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + return (vocab_file,) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 62623b4066cb..1f5931885f83 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -4219,6 +4219,30 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class MgpstrForSceneTextRecognition(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MgpstrModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MgpstrPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class MMBTForClassification(metaclass=DummyObject): _backends = ["torch"] diff --git a/tests/models/mgp_str/__init__.py b/tests/models/mgp_str/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/mgp_str/test_modeling_mgp_str.py b/tests/models/mgp_str/test_modeling_mgp_str.py new file mode 100644 index 000000000000..ab5b6422b627 --- /dev/null +++ b/tests/models/mgp_str/test_modeling_mgp_str.py @@ -0,0 +1,269 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch MGP-STR model. """ + +import inspect +import unittest + +import requests + +from transformers import MgpstrConfig +from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.utils import is_torch_available, is_vision_available + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor + + +if is_torch_available(): + import torch + from torch import nn + + from transformers import MgpstrForSceneTextRecognition + + +if is_vision_available(): + from PIL import Image + + from transformers import MgpstrProcessor + + +class MgpstrModelTester: + def __init__( + self, + parent, + is_training=False, + batch_size=13, + image_size=(32, 128), + patch_size=4, + num_channels=3, + max_token_length=27, + num_character_labels=38, + num_bpe_labels=99, + num_wordpiece_labels=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + mlp_ratio=4.0, + patch_embeds_hidden_size=257, + output_hidden_states=None, + ): + self.parent = parent + self.is_training = is_training + self.batch_size = batch_size + self.image_size = image_size + self.patch_size = patch_size + self.num_channels = num_channels + self.max_token_length = max_token_length + self.num_character_labels = num_character_labels + self.num_bpe_labels = num_bpe_labels + self.num_wordpiece_labels = num_wordpiece_labels + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.mlp_ratio = mlp_ratio + self.patch_embeds_hidden_size = patch_embeds_hidden_size + self.output_hidden_states = output_hidden_states + + def prepare_config_and_inputs(self): + pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]) + config = self.get_config() + return config, pixel_values + + def get_config(self): + return MgpstrConfig( + image_size=self.image_size, + patch_size=self.patch_size, + num_channels=self.num_channels, + max_token_length=self.max_token_length, + num_character_labels=self.num_character_labels, + num_bpe_labels=self.num_bpe_labels, + num_wordpiece_labels=self.num_wordpiece_labels, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + mlp_ratio=self.mlp_ratio, + output_hidden_states=self.output_hidden_states, + ) + + def create_and_check_model(self, config, pixel_values): + model = MgpstrForSceneTextRecognition(config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + generated_ids = model(pixel_values) + self.parent.assertEqual( + generated_ids[0][0].shape, (self.batch_size, self.max_token_length, self.num_character_labels) + ) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, pixel_values = config_and_inputs + inputs_dict = {"pixel_values": pixel_values} + return config, inputs_dict + + +@require_torch +class MgpstrModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = (MgpstrForSceneTextRecognition,) if is_torch_available() else () + fx_compatible = False + + test_pruning = False + test_resize_embeddings = False + test_head_masking = False + test_attention_outputs = False + + def setUp(self): + self.model_tester = MgpstrModelTester(self) + self.config_tester = ConfigTester(self, config_class=MgpstrConfig, has_text_modality=False) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + @unittest.skip(reason="MgpstrModel does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + def test_model_common_attributes(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + self.assertIsInstance(model.get_input_embeddings(), (nn.Module)) + x = model.get_output_embeddings() + self.assertTrue(x is None or isinstance(x, nn.Linear)) + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["pixel_values"] + self.assertListEqual(arg_names[:1], expected_arg_names) + + @unittest.skip(reason="MgpstrModel does not support feedforward chunking") + def test_feed_forward_chunking(self): + pass + + def test_gradient_checkpointing_backward_compatibility(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + if not model_class.supports_gradient_checkpointing: + continue + + config.gradient_checkpointing = True + model = model_class(config) + self.assertTrue(model.is_gradient_checkpointing) + + def test_hidden_states_output(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(config) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + + hidden_states = outputs.hidden_states + + expected_num_layers = getattr( + self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1 + ) + self.assertEqual(len(hidden_states), expected_num_layers) + + self.assertListEqual( + list(hidden_states[0].shape[-2:]), + [self.model_tester.patch_embeds_hidden_size, self.model_tester.hidden_size], + ) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + # override as the `logit_scale` parameter initilization is different for MgpstrModel + def test_initialization(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + configs_no_init = _config_zero_init(config) + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + for name, param in model.named_parameters(): + if isinstance(param, (nn.Linear, nn.Conv2d, nn.LayerNorm)): + if param.requires_grad: + self.assertIn( + ((param.data.mean() * 1e9).round() / 1e9).item(), + [0.0, 1.0], + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + + @unittest.skip(reason="Retain_grad is tested in individual model tests") + def test_retain_grad_hidden_states_attentions(self): + pass + + +# We will verify our results on an image from the IIIT-5k dataset +def prepare_img(): + url = "https://i.postimg.cc/ZKwLg2Gw/367-14.png" + im = Image.open(requests.get(url, stream=True).raw).convert("RGB") + return im + + +@require_vision +@require_torch +class MgpstrModelIntegrationTest(unittest.TestCase): + @slow + def test_inference(self): + model_name = "alibaba-damo/mgp-str-base" + model = MgpstrForSceneTextRecognition.from_pretrained(model_name).to(torch_device) + processor = MgpstrProcessor.from_pretrained(model_name) + + image = prepare_img() + inputs = processor(images=image, return_tensors="pt").pixel_values.to(torch_device) + + # forward pass + with torch.no_grad(): + outputs = model(inputs) + + # verify the logits + self.assertEqual(outputs.logits[0].shape, torch.Size((1, 27, 38))) + + out_strs = processor.batch_decode(outputs.logits) + expected_text = "ticket" + + self.assertEqual(out_strs["generated_text"][0], expected_text) + + expected_slice = torch.tensor( + [[[-39.7358, -44.8562, -36.6253], [-62.3605, -64.5908, -59.0069], [-74.6127, -68.9724, -71.7150]]], + device=torch_device, + ) + + self.assertTrue(torch.allclose(outputs.logits[0][:, 1:4, 1:4], expected_slice, atol=1e-4)) diff --git a/tests/models/mgp_str/test_processor_mgp_str.py b/tests/models/mgp_str/test_processor_mgp_str.py new file mode 100644 index 000000000000..387d13fad18e --- /dev/null +++ b/tests/models/mgp_str/test_processor_mgp_str.py @@ -0,0 +1,211 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the MgpstrProcessor. """ + +import json +import os +import shutil +import tempfile +import unittest + +import numpy as np +import pytest + +from transformers import MgpstrTokenizer +from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import IMAGE_PROCESSOR_NAME, is_torch_available, is_vision_available + + +if is_torch_available(): + import torch + + +if is_vision_available(): + from PIL import Image + + from transformers import MgpstrProcessor, ViTImageProcessor + + +@require_torch +@require_vision +class MgpstrProcessorTest(unittest.TestCase): + image_processing_class = ViTImageProcessor if is_vision_available() else None + + @property + def image_processor_dict(self): + return self.image_processor_tester.prepare_image_processor_dict() + + def setUp(self): + self.image_size = (3, 32, 128) + self.tmpdirname = tempfile.mkdtemp() + + # fmt: off + vocab = ['[GO]', '[s]', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] + # fmt: on + vocab_tokens = dict(zip(vocab, range(len(vocab)))) + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as fp: + fp.write(json.dumps(vocab_tokens) + "\n") + + image_processor_map = { + "do_normalize": False, + "do_resize": True, + "feature_extractor_type": "ViTFeatureExtractor", + "resample": 3, + "size": {"height": 32, "width": 128}, + } + self.image_processor_file = os.path.join(self.tmpdirname, IMAGE_PROCESSOR_NAME) + with open(self.image_processor_file, "w", encoding="utf-8") as fp: + json.dump(image_processor_map, fp) + + def get_tokenizer(self, **kwargs): + return MgpstrTokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_image_processor(self, **kwargs): + return ViTImageProcessor.from_pretrained(self.tmpdirname, **kwargs) + + def tearDown(self): + shutil.rmtree(self.tmpdirname) + + def prepare_image_inputs(self): + """This function prepares a list of PIL images.""" + + image_input = np.random.randint(255, size=(3, 30, 400), dtype=np.uint8) + + image_input = Image.fromarray(np.moveaxis(image_input, 0, -1)) + + return image_input + + def test_save_load_pretrained_default(self): + tokenizer = self.get_tokenizer() + image_processor = self.get_image_processor() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + processor.save_pretrained(self.tmpdirname) + processor = MgpstrProcessor.from_pretrained(self.tmpdirname, use_fast=False) + + self.assertEqual(processor.char_tokenizer.get_vocab(), tokenizer.get_vocab()) + self.assertIsInstance(processor.char_tokenizer, MgpstrTokenizer) + + self.assertEqual(processor.image_processor.to_json_string(), image_processor.to_json_string()) + self.assertIsInstance(processor.image_processor, ViTImageProcessor) + + def test_save_load_pretrained_additional_features(self): + tokenizer = self.get_tokenizer() + image_processor = self.get_image_processor() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + processor.save_pretrained(self.tmpdirname) + + tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)") + image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0) + + processor = MgpstrProcessor.from_pretrained( + self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0 + ) + + self.assertEqual(processor.char_tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab()) + self.assertIsInstance(processor.char_tokenizer, MgpstrTokenizer) + + self.assertEqual(processor.image_processor.to_json_string(), image_processor_add_kwargs.to_json_string()) + self.assertIsInstance(processor.image_processor, ViTImageProcessor) + + def test_image_processor(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + + image_input = self.prepare_image_inputs() + + input_image_proc = image_processor(image_input, return_tensors="np") + input_processor = processor(images=image_input, return_tensors="np") + + for key in input_image_proc.keys(): + self.assertAlmostEqual(input_image_proc[key].sum(), input_processor[key].sum(), delta=1e-2) + + def test_tokenizer(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + + input_str = "test" + + encoded_processor = processor(text=input_str) + + encoded_tok = tokenizer(input_str) + for key in encoded_tok.keys(): + self.assertListEqual(encoded_tok[key], encoded_processor[key]) + + def test_processor(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + + input_str = "test" + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input) + + self.assertListEqual(list(inputs.keys()), ["pixel_values", "labels"]) + + # test if it raises when no input is passed + with pytest.raises(ValueError): + processor() + + def test_tokenizer_decode(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + + predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9], [3, 4, 3, 1, 1, 8, 9]] + + decoded_processor = processor.char_decode(predicted_ids) + decoded_tok = tokenizer.batch_decode(predicted_ids) + decode_strs = [seq.replace(" ", "") for seq in decoded_tok] + + self.assertListEqual(decode_strs, decoded_processor) + + def test_model_input_names(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + + input_str = None + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input) + + self.assertListEqual(list(inputs.keys()), processor.model_input_names) + + def test_processor_batch_decode(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = MgpstrProcessor(tokenizer=tokenizer, image_processor=image_processor) + + char_input = torch.randn(1, 27, 38) + bpe_input = torch.randn(1, 27, 50257) + wp_input = torch.randn(1, 27, 30522) + + results = processor.batch_decode([char_input, bpe_input, wp_input]) + + self.assertListEqual(list(results.keys()), ["generated_text", "scores", "char_preds", "bpe_preds", "wp_preds"]) diff --git a/tests/models/mgp_str/test_tokenization_mgp_str.py b/tests/models/mgp_str/test_tokenization_mgp_str.py new file mode 100644 index 000000000000..a05d7f3cbf90 --- /dev/null +++ b/tests/models/mgp_str/test_tokenization_mgp_str.py @@ -0,0 +1,96 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import os +import unittest + +from transformers import MgpstrTokenizer +from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES +from transformers.testing_utils import require_tokenizers + +from ...test_tokenization_common import TokenizerTesterMixin + + +@require_tokenizers +class MgpstrTokenizationTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = MgpstrTokenizer + test_rust_tokenizer = False + from_pretrained_kwargs = {} + test_seq2seq = False + + def setUp(self): + super().setUp() + + # fmt: off + vocab = ['[GO]', '[s]', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] + # fmt: on + vocab_tokens = dict(zip(vocab, range(len(vocab)))) + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as fp: + fp.write(json.dumps(vocab_tokens) + "\n") + + def get_tokenizer(self, **kwargs): + return MgpstrTokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self, tokenizer): + input_text = "tester" + output_text = "tester" + return input_text, output_text + + @unittest.skip("MGP-STR always lower cases letters.") + def test_added_tokens_do_lower_case(self): + pass + + def test_add_special_tokens(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + special_token = "[SPECIAL_TOKEN]" + + tokenizer.add_special_tokens({"cls_token": special_token}) + encoded_special_token = tokenizer.encode([special_token], add_special_tokens=False) + self.assertEqual(len(encoded_special_token), 1) + + decoded = tokenizer.decode(encoded_special_token, skip_special_tokens=True) + self.assertTrue(special_token not in decoded) + + def test_internal_consistency(self): + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + input_text, output_text = self.get_input_output_texts(tokenizer) + + tokens = tokenizer.tokenize(input_text) + ids = tokenizer.convert_tokens_to_ids(tokens) + ids_2 = tokenizer.encode(input_text, add_special_tokens=False) + self.assertListEqual(ids, ids_2) + + tokens_2 = tokenizer.convert_ids_to_tokens(ids) + self.assertNotEqual(len(tokens_2), 0) + text_2 = tokenizer.decode(ids) + self.assertIsInstance(text_2, str) + + self.assertEqual(text_2.replace(" ", ""), output_text) + + @unittest.skip("MGP-STR tokenizer only handles one sequence.") + def test_maximum_encoding_length_pair_input(self): + pass + + @unittest.skip("inputs cannot be pretokenized in MgpstrTokenizer") + def test_pretokenized_inputs(self): + pass diff --git a/utils/check_repo.py b/utils/check_repo.py index f16c4fb851bf..fca0cd86ef90 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -95,6 +95,7 @@ "M2M100Encoder", # Building part of bigger (tested) model. "M2M100Decoder", # Building part of bigger (tested) model. "MCTCTEncoder", # Building part of bigger (tested) model. + "MgpstrModel", # Building part of bigger (tested) model. "Speech2TextEncoder", # Building part of bigger (tested) model. "Speech2TextDecoder", # Building part of bigger (tested) model. "LEDEncoder", # Building part of bigger (tested) model. @@ -269,6 +270,7 @@ "LukeForEntityClassification", "LukeForEntityPairClassification", "LukeForEntitySpanClassification", + "MgpstrModel", "OpenAIGPTDoubleHeadsModel", "OwlViTTextModel", "OwlViTVisionModel", From c211bd593e1621218839b601421feec612dbfdbf Mon Sep 17 00:00:00 2001 From: Alex Calabrese <58480609+alexcalabrese@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:24:34 +0100 Subject: [PATCH 066/277] Add pr_checks.mdx Italian translation (#17459) (#22116) * Add pr_checks.mdx Italian translation (#17459) * Updated pr_checks.mdx Italian translation (#17459) --- docs/source/it/_toctree.yml | 2 + docs/source/it/pr_checks.mdx | 131 +++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 docs/source/it/pr_checks.mdx diff --git a/docs/source/it/_toctree.yml b/docs/source/it/_toctree.yml index 16d36f26e730..6d4b5e1f2ae1 100644 --- a/docs/source/it/_toctree.yml +++ b/docs/source/it/_toctree.yml @@ -45,5 +45,7 @@ title: Hardware ottimizzato per l'addestramento - local: community title: Risorse della comunità + - local: pr_checks + title: Controlli su una Pull Request title: Guide How-to diff --git a/docs/source/it/pr_checks.mdx b/docs/source/it/pr_checks.mdx new file mode 100644 index 000000000000..d7541d59f0ad --- /dev/null +++ b/docs/source/it/pr_checks.mdx @@ -0,0 +1,131 @@ + + +# Controlli su una Pull Request + +Quando apri una pull request sui 🤗 Transformers, vengono eseguiti un discreto numero di controlli per assicurarsi che la patch che stai aggiungendo non stia rompendo qualcosa di esistente. Questi controlli sono di quattro tipi: +- test regolari +- costruzione della documentazione +- stile del codice e della documentazione +- coerenza generale del repository + +In questo documento, cercheremo di spiegare quali sono i vari controlli e le loro ragioni, oltre a spiegare come eseguire il debug locale se uno di essi fallisce sulla tua PR. + +Nota che tutti richiedono un'installazione dev: + +```bash +pip install transformers[dev] +``` + +o un'installazione modificabile: + +```bash +pip install -e .[dev] +``` + +all'interno del repo Transformers. + +## Tests + +Tutti i job che iniziano con `ci/circleci: run_tests_` eseguono parti della suite di test dei Transformers. Ognuno di questi job si concentra su una parte della libreria in un determinato ambiente: per esempio `ci/circleci: run_tests_pipelines_tf` esegue il test delle pipeline in un ambiente in cui è installato solo TensorFlow. + +Nota che per evitare di eseguire i test quando non ci sono cambiamenti reali nei moduli che si stanno testando, ogni volta viene eseguita solo una parte della suite di test: viene eseguita una utility per determinare le differenze nella libreria tra prima e dopo la PR (ciò che GitHub mostra nella scheda "Files changes") e sceglie i test che sono stati impattati dalla diff. Questa utility può essere eseguita localmente con: + +```bash +python utils/tests_fetcher.py +``` + +dalla root del repo Transformers. Di seguito ciò che farà: + +1. Controlla per ogni file nel diff se le modifiche sono nel codice o solo nei commenti o nelle docstrings. Vengono mantenuti solo i file con modifiche reali al codice. +2. Costruisce una mappa interna che fornisce per ogni file del codice sorgente della libreria tutti i file su cui ha un impatto ricorsivo. Si dice che il modulo A ha un impatto sul modulo B se il modulo B importa il modulo A. Per l'impatto ricorsivo, abbiamo bisogno di una catena di moduli che va dal modulo A al modulo B in cui ogni modulo importa il precedente. +3. Applica questa mappa ai file raccolti nel passaggio 1, si ottiene l'elenco dei file del modello interessati dalla PR. +4. Mappa ciascuno di questi file con i corrispondenti file di test e ottiene l'elenco dei test da eseguire. + +Quando esegui lo script in locale, dovresti ottenere la stampa dei risultati dei passi 1, 3 e 4 e quindi sapere quali test sono stati eseguiti. Lo script creerà anche un file chiamato `test_list.txt` che contiene l'elenco dei test da eseguire e che puoi eseguire localmente con il seguente comando: + +```bash +python -m pytest -n 8 --dist=loadfile -rA -s $(cat test_list.txt) +``` + +Nel caso in cui qualcosa sia sfuggito, l'intera suite di test viene eseguita quotidianamente. + +## Build della documentazione + +Il job `ci/circleci: build_doc` esegue una build della documentazione per assicurarsi che tutto sia a posto una volta che la PR è stata unita. Se questo passaggio fallisce, puoi controllare localmente entrando nella cartella `docs` del repo Transformers e digitare + +```bash +make html +``` + +Sphinx non è noto per i suoi messaggi di errore chiari, quindi potrebbe essere necessario che provi alcune cose per trovare davvero la fonte dell'errore. + +## Stile del codice e della documentazione + +La formattazione del codice viene applicata a tutti i file sorgenti, agli esempi e ai test usando `black` e `isort`. Abbiamo anche uno strumento personalizzato che si occupa della formattazione delle docstring e dei file `rst` (`utils/style_doc.py`), così come dell'ordine dei lazy imports eseguiti nei file `__init__.py` dei Transformers (`utils/custom_init_isort.py`). Tutto questo può essere lanciato eseguendo + +```bash +make style +``` + +I controlli della CI sono applicati all'interno del controllo `ci/circleci: check_code_quality`. Esegue anche `flake8`, che dà un'occhiata di base al codice e si lamenta se trova una variabile non definita o non utilizzata. Per eseguire questo controllo localmente, usare + +```bash +make quality +``` + +Questa operazione può richiedere molto tempo, quindi per eseguire la stessa operazione solo sui file modificati nel branch corrente, eseguire + +```bash +make fixup +``` + +Quest'ultimo comando eseguirà anche tutti i controlli aggiuntivi per la consistenza del repository. Diamogli un'occhiata. + +## Coerenza del repository + +All'interno sono raggruppati tutti i test per assicurarsi che la tua PR lasci il repository in un buono stato ed è eseguito dal controllo `ci/circleci: check_repository_consistency`. Puoi eseguire localmente questo controllo eseguendo quanto segue: + +```bash +make repo-consistency +``` + +Questo verifica che: + +- Tutti gli oggetti aggiunti all'init sono documentati (eseguito da `utils/check_repo.py`) +- Tutti i file `__init__.py` hanno lo stesso contenuto nelle loro due sezioni (eseguito da `utils/check_inits.py`) +- Tutto il codice identificato come copia da un altro modulo è coerente con l'originale (eseguito da `utils/check_copies.py`) +- Le traduzioni dei README e l'indice della documentazione hanno lo stesso elenco di modelli del README principale (eseguito da `utils/check_copies.py`) +- Le tabelle autogenerate nella documentazione sono aggiornate (eseguito da `utils/check_table.py`) +- La libreria ha tutti gli oggetti disponibili anche se non tutte le dipendenze opzionali sono installate (eseguito da `utils/check_dummies.py`) + +Se questo controllo fallisce, le prime due voci richiedono una correzione manuale, mentre le ultime quattro possono essere corrette automaticamente per te eseguendo il comando + +```bash +make fix-copies +``` + +Ulteriori controlli riguardano le PR che aggiungono nuovi modelli, principalmente che: + +- Tutti i modelli aggiunti sono in un Auto-mapping (eseguita da `utils/check_repo.py`) + +- Tutti i modelli sono testati correttamente (eseguito da `utils/check_repo.py`) + + \ No newline at end of file From 6048e88c2a4065e64f83b88625a2685aa9b971f3 Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Mon, 13 Mar 2023 13:49:23 +0000 Subject: [PATCH 067/277] Fix gradient checkpointing bug in xglm (#22127) --- src/transformers/models/xglm/modeling_xglm.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/xglm/modeling_xglm.py b/src/transformers/models/xglm/modeling_xglm.py index d8e9952ee798..b9cef18efc7e 100755 --- a/src/transformers/models/xglm/modeling_xglm.py +++ b/src/transformers/models/xglm/modeling_xglm.py @@ -714,6 +714,14 @@ def forward( hidden_states = nn.functional.dropout(hidden_states, p=float(self.dropout), training=self.training) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache = True` is incompatible with gradient checkpointing`. Setting `use_cache =" + " False`..." + ) + use_cache = False + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -739,12 +747,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning( - "`use_cache = True` is incompatible with gradient checkpointing`. Setting `use_cache =" - " False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 27ca3663b5643c8267922a08dbc21cd6faf29516 Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Mon, 13 Mar 2023 13:50:02 +0000 Subject: [PATCH 068/277] Fix gradient checkpointing bug in Trajectory Transformer (#22125) --- .../modeling_trajectory_transformer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/trajectory_transformer/modeling_trajectory_transformer.py b/src/transformers/models/trajectory_transformer/modeling_trajectory_transformer.py index 682f814c9983..e8ecedccb5ea 100644 --- a/src/transformers/models/trajectory_transformer/modeling_trajectory_transformer.py +++ b/src/transformers/models/trajectory_transformer/modeling_trajectory_transformer.py @@ -533,6 +533,13 @@ def forward( hidden_states = self.drop(token_embeddings + position_embeddings) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + presents = () if use_cache else None all_self_attentions = () if output_attentions else None all_hidden_states = () if output_hidden_states else None @@ -542,11 +549,6 @@ def forward( all_hidden_states = all_hidden_states + (hidden_states,) if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 3e3234549b82dada0bf32e54f66d85e595cfd66f Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Mon, 13 Mar 2023 13:52:34 +0000 Subject: [PATCH 069/277] Fix gradient checkpointing bug in xlm_roberta_xl (#22128) --- .../models/xlm_roberta_xl/modeling_xlm_roberta_xl.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py index ad249f0835b1..e96e20f89695 100644 --- a/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py +++ b/src/transformers/models/xlm_roberta_xl/modeling_xlm_roberta_xl.py @@ -478,6 +478,12 @@ def forward( output_hidden_states=False, return_dict=True, ): + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None @@ -491,11 +497,6 @@ def forward( past_key_value = past_key_values[i] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 9c13edb79a81cdc2d401555af48ec641753d2c16 Mon Sep 17 00:00:00 2001 From: Nicola Procopio Date: Mon, 13 Mar 2023 15:02:03 +0100 Subject: [PATCH 070/277] Added big_models.mdx italian translation #17600 (#22115) * updated toctree * italian translation big_model.mdx * italian translation big_models --- docs/source/it/_toctree.yml | 2 + docs/source/it/big_models.mdx | 119 ++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 docs/source/it/big_models.mdx diff --git a/docs/source/it/_toctree.yml b/docs/source/it/_toctree.yml index 6d4b5e1f2ae1..86c3465b478c 100644 --- a/docs/source/it/_toctree.yml +++ b/docs/source/it/_toctree.yml @@ -33,6 +33,8 @@ title: Convertire modelli tensorflow - local: serialization title: Esporta modelli Transformers + - local: big_models + title: Istanziare un big model - local: debugging title: Debugging title: Guide pratiche diff --git a/docs/source/it/big_models.mdx b/docs/source/it/big_models.mdx new file mode 100644 index 000000000000..56a0fa6fea4a --- /dev/null +++ b/docs/source/it/big_models.mdx @@ -0,0 +1,119 @@ + + +# Istanziare un big model + +Quando vuoi utilizzare un modello preaddestrato (pretrained) molto grande, una sfida è minimizzare l'uso della RAM. Il workflow classico +in PyTorch è: + +1. Crea il tuo modello con pesi casuali (random weights). +2. Carica i tuoi pesi preaddestrati. +3. Inserisci i pesi preaddestrati nel tuo modello casuale. + +I passi 1 e 2 una versione completa del modello in memoria, in molti casi non è un problema, ma se il modello inizia a pesare diversi GigaBytes, queste due copie possono sturare la nostra RAM. Ancora peggio, se stai usando `torch.distributed` per seguire l'addestramento (training) in distribuito, ogni processo caricherà il modello preaddestrato e memorizzerà queste due copie nella RAM. + + + +Nota che il modello creato casualmente è inizializzato con tensori "vuoti", che occupano spazio in memoria ma senza riempirlo (quindi i valori casuali sono quelli che si trovavano in questa porzione di memoria in un determinato momento). L'inizializzazione casuale che segue la distribuzione appropriata per il tipo di modello/parametri istanziato (come la distribuzione normale per le istanze) è eseguito solo dopo il passaggio 3 sui pesi non inizializzati, per essere più rapido possibile! + + + +In questa guida, esploreremo le soluzioni che Transformers offre per affrontare questo problema. C'è da tenere in conto che questa è un'area in cui si sta attualmente sviluppando, quindi le API spiegate qui possono variare velocemente in futuro. + +## Checkpoints condivisi + +Dalla versione 4.18.0, i checkpoints dei modelli che occupano più di 10GB di spazio vengono automaticamente frammentati in più parti. Per quanto riguarda la possibilità di avere un unico checkpoint quando si utilizza `model.save_pretrained(save_dir)`, si hanno diversi checkpoint parziali (ognuno con dimensione < 10GB) e un indice che mappa i nomi dei parametri ai file in cui sono memorizzati. + +Puoi controllare la dimensione massima dopo la frammentazione con il parametro `max_shard_size`, nel prossimo esempio, useremo modelli di dimensioni normali con frammenti di piccoli dimensioni: prendiamo un modello BERT classico. + +```py +from transformers import AutoModel + +model = AutoModel.from_pretrained("bert-base-cased") +``` + +Se tu salvi usando [`~PreTrainedModel.save_pretrained`], avrai una nuova cartella con due file: il config del modello e i suoi pesi: + +```py +>>> import os +>>> import tempfile + +>>> with tempfile.TemporaryDirectory() as tmp_dir: +... model.save_pretrained(tmp_dir) +... print(sorted(os.listdir(tmp_dir))) +['config.json', 'pytorch_model.bin'] +``` + +Adesso usiamo una dimensione massima di frammentazione di 200MB: + +```py +>>> with tempfile.TemporaryDirectory() as tmp_dir: +... model.save_pretrained(tmp_dir, max_shard_size="200MB") +... print(sorted(os.listdir(tmp_dir))) +['config.json', 'pytorch_model-00001-of-00003.bin', 'pytorch_model-00002-of-00003.bin', 'pytorch_model-00003-of-00003.bin', 'pytorch_model.bin.index.json'] +``` + +In aggiunta alla configurazione del modello, vediamo tre differenti file dei pesi, e un file `index.json` che è il nostro indice. Un checkpoint può essere ricaricato totalmente usando il metodo [`~PreTrainedModel.from_pretrained`]: + +```py +>>> with tempfile.TemporaryDirectory() as tmp_dir: +... model.save_pretrained(tmp_dir, max_shard_size="200MB") +... new_model = AutoModel.from_pretrained(tmp_dir) +``` + +Il vantaggio principale di applicare questo metodo per modelli grandi è che durante il passo 2 del workflow illustrato in precedenza, ogni frammento del checkpoint viene caricato dopo il precedente, limitando l'utilizzo della RAM alla dimensione del modello più la dimensione del frammento più grande. + +Dietro le quinte, il file indice è utilizzato per determinare quali chiavi sono nel checkpoint, e dove i corrispondenti pesi sono memorizzati. Possiamo caricare l'indice come un qualsiasi json e ottenere un dizionario: + +```py +>>> import json + +>>> with tempfile.TemporaryDirectory() as tmp_dir: +... model.save_pretrained(tmp_dir, max_shard_size="200MB") +... with open(os.path.join(tmp_dir, "pytorch_model.bin.index.json"), "r") as f: +... index = json.load(f) + +>>> print(index.keys()) +dict_keys(['metadata', 'weight_map']) +``` + +I metadati consistono solo nella dimensione totale del modello per ora. Abbiamo in programma di aggiungere altre informazioni in futuro: + +```py +>>> index["metadata"] +{'total_size': 433245184} +``` + +La mappa dei pesi è la parte principale di questo indice, che mappa ogni nome dei parametri (si trova solitamente nei modelli PyTorch come `state_dict`) al file in cui è memorizzato: + +```py +>>> index["weight_map"] +{'embeddings.LayerNorm.bias': 'pytorch_model-00001-of-00003.bin', + 'embeddings.LayerNorm.weight': 'pytorch_model-00001-of-00003.bin', + ... +``` + +Se vuoi caricare direttamente un checkpoint frammentato in un modello senza usare [`~PreTrainedModel.from_pretrained`] (come si farebbe con `model.load_state_dict()` per un checkpoint completo) devi usare [`~modeling_utils.load_sharded_checkpoint`]: + +```py +>>> from transformers.modeling_utils import load_sharded_checkpoint + +>>> with tempfile.TemporaryDirectory() as tmp_dir: +... model.save_pretrained(tmp_dir, max_shard_size="200MB") +... load_sharded_checkpoint(model, tmp_dir) +``` + +## Caricamento low memory + +Frammentare i checkpoint l'utilizzo di memoria al passo 2 del workflow citato in precedenza, ma per utilizzare questo modello in un ambiente con poca memoria, consigliamo di utilizzare i nostri strumenti basati sulla libreria Accelerate. + +Per ulteriori informazioni, leggere la seguente guida: [Large model loading using Accelerate](./main_classes/model#large-model-loading) \ No newline at end of file From 718ec35d43bf85943e3cf95ccae2de870fbb507a Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:03:21 +0100 Subject: [PATCH 071/277] [`Blip2`] skip accelerate test (#22124) skip accelerate test --- tests/models/blip_2/test_modeling_blip_2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 302feb87a117..f31d1624a471 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -710,6 +710,10 @@ def test_save_load_fast_init_from_base(self): def test_save_load_fast_init_to_base(self): pass + @unittest.skip(reason="Does not work on the tiny model as we keep hitting edge cases.") + def test_cpu_offload(self): + pass + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() From d56076dbcd2cc2cb2f3a08426ccc433ee67db9a1 Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:05:11 +0000 Subject: [PATCH 072/277] Fix gradient checkpointing bug in xmod (#22129) --- src/transformers/models/xmod/modeling_xmod.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/xmod/modeling_xmod.py b/src/transformers/models/xmod/modeling_xmod.py index 8e1325e4d072..6453701fe93c 100644 --- a/src/transformers/models/xmod/modeling_xmod.py +++ b/src/transformers/models/xmod/modeling_xmod.py @@ -552,6 +552,12 @@ def forward( output_hidden_states: Optional[bool] = False, return_dict: Optional[bool] = True, ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPastAndCrossAttentions]: + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None @@ -565,11 +571,6 @@ def forward( past_key_value = past_key_values[i] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From b22b7b16b4b8aed3a3754c92997290d0fddfc63f Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:06:17 +0000 Subject: [PATCH 073/277] Fix gradient checkpointing bug in LongT5 (#22130) --- src/transformers/models/longt5/modeling_longt5.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/longt5/modeling_longt5.py b/src/transformers/models/longt5/modeling_longt5.py index 366fbc4f7c17..e06743b78d96 100644 --- a/src/transformers/models/longt5/modeling_longt5.py +++ b/src/transformers/models/longt5/modeling_longt5.py @@ -1480,6 +1480,13 @@ def forward( else: encoder_extended_attention_mask = None + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + # Prepare head mask if needed head_mask = self.get_head_mask(head_mask, self.config.num_layers) cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers) @@ -1500,8 +1507,6 @@ def forward( all_hidden_states = all_hidden_states + (hidden_states,) if self.gradient_checkpointing and self.training: - if use_cache: - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From c349eda43fea8ea15c9d56b287d46e3de5b608d4 Mon Sep 17 00:00:00 2001 From: Karim Foda <35491698+KMFODA@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:45:47 +0000 Subject: [PATCH 074/277] Fix gradient checkpointing bug in trocr (#22126) * Fix gradient checkpointing bug in trocr * Fix format * Update src/transformers/models/trocr/modeling_trocr.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --------- Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- src/transformers/models/trocr/modeling_trocr.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/trocr/modeling_trocr.py b/src/transformers/models/trocr/modeling_trocr.py index 5eda7479b4d1..e6853d0c5a8e 100644 --- a/src/transformers/models/trocr/modeling_trocr.py +++ b/src/transformers/models/trocr/modeling_trocr.py @@ -664,6 +664,13 @@ def forward( # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] encoder_attention_mask = _expand_mask(encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..." + ) + use_cache = False + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -689,12 +696,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning( - "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache =" - " False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 165bd4a865d85f9837672e49d60dc98b8e6e2ba6 Mon Sep 17 00:00:00 2001 From: Maria Khalusova Date: Mon, 13 Mar 2023 10:57:17 -0400 Subject: [PATCH 075/277] Zero-shot image classification task guide (#22132) * WIP * WIP * manual inference example * make style * Apply suggestions from code review Co-authored-by: Alara Dirik <8944735+alaradirik@users.noreply.github.com> --------- Co-authored-by: Alara Dirik <8944735+alaradirik@users.noreply.github.com> --- docs/source/en/_toctree.yml | 2 + .../tasks/zero_shot_image_classification.mdx | 143 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 docs/source/en/tasks/zero_shot_image_classification.mdx diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index f164348a94aa..eddfc48d5cd5 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -83,6 +83,8 @@ title: Object detection - local: tasks/zero_shot_object_detection title: Zero-shot object detection + - local: tasks/zero_shot_image_classification + title: Zero-shot image classification title: Computer Vision - sections: - local: tasks/image_captioning diff --git a/docs/source/en/tasks/zero_shot_image_classification.mdx b/docs/source/en/tasks/zero_shot_image_classification.mdx new file mode 100644 index 000000000000..2784d365083f --- /dev/null +++ b/docs/source/en/tasks/zero_shot_image_classification.mdx @@ -0,0 +1,143 @@ + + +# Zero-shot image classification + +[[open-in-colab]] + +Zero-shot image classification is a task that involves classifying images into different categories using a model that was +not explicitly trained on data containing labeled examples from those specific categories. + +Traditionally, image classification requires training a model on a specific set of labeled images, and this model learns to +"map" certain image features to labels. When there's a need to use such model for a classification task that introduces a +new set of labels, fine-tuning is required to "recalibrate" the model. + +In contrast, zero-shot or open vocabulary image classification models are typically multi-modal models that have been trained on a large +dataset of images and associated descriptions. These models learn aligned vision-language representations that can be used for many downstream tasks including zero-shot image classification. + +This is a more flexible approach to image classification that allows models to generalize to new and unseen categories +without the need for additional training data and enables users to query images with free-form text descriptions of their target objects . + +In this guide you'll learn how to: + +* create a zero-shot image classification pipeline +* run zero-shot image classification inference by hand + +Before you begin, make sure you have all the necessary libraries installed: + +```bash +pip install -q transformers +``` + +## Zero-shot image classification pipeline + +The simplest way to try out inference with a model supporting zero-shot image classification is to use the corresponding [`pipeline`]. +Instantiate a pipeline from a [checkpoint on the Hugging Face Hub](https://huggingface.co/models?pipeline_tag=zero-shot-image-classification&sort=downloads): + +```python +>>> from transformers import pipeline + +>>> checkpoint = "openai/clip-vit-large-patch14" +>>> detector = pipeline(model=checkpoint, task="zero-shot-image-classification") +``` + +Next, choose an image you'd like to classify. + +```py +>>> from PIL import Image +>>> import requests + +>>> url = "https://unsplash.com/photos/g8oS8-82DxI/download?ixid=MnwxMjA3fDB8MXx0b3BpY3x8SnBnNktpZGwtSGt8fHx8fDJ8fDE2NzgxMDYwODc&force=true&w=640" +>>> image = Image.open(requests.get(url, stream=True).raw) + +>>> image +``` + +
+ Photo of an owl +
+ +Pass the image and the candidate object labels to the pipeline. Here we pass the image directly; other suitable options +include a local path to an image or an image url. +The candidate labels can be simple words like in this example, or more descriptive. + +```py +>>> predictions = classifier(image, candidate_labels=["fox", "bear", "seagull", "owl"]) +>>> predictions +[{'score': 0.9996670484542847, 'label': 'owl'}, + {'score': 0.000199399160919711, 'label': 'seagull'}, + {'score': 7.392891711788252e-05, 'label': 'fox'}, + {'score': 5.96074532950297e-05, 'label': 'bear'}] +``` + +## Zero-shot image classification by hand + +Now that you've seen how to use the zero-shot image classification pipeline, let's take a look how you can run zero-shot +image classification manually. + +Start by loading the model and associated processor from a [checkpoint on the Hugging Face Hub](https://huggingface.co/models?pipeline_tag=zero-shot-image-classification&sort=downloads). +Here we'll use the same checkpoint as before: + +```py +>>> from transformers import AutoProcessor, AutoModelForZeroShotImageClassification + +>>> model = AutoModelForZeroShotImageClassification.from_pretrained(checkpoint) +>>> processor = AutoProcessor.from_pretrained(checkpoint) +``` + +Let's take a different image to switch things up. + +```py +>>> from PIL import Image +>>> import requests + +>>> url = "https://unsplash.com/photos/xBRQfR2bqNI/download?ixid=MnwxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNjc4Mzg4ODEx&force=true&w=640" +>>> image = Image.open(requests.get(url, stream=True).raw) + +>>> image +``` + +
+ Photo of a car +
+ +Use the processor to prepare the inputs for the model. The processor combines an image processor that prepares the +image for the model by resizing and normalizing it, and a tokenizer that takes care of the text inputs. + +```py +>>> candidate_labels = ["tree", "car", "bike", "cat"] +>>> inputs = processor(images=image, text=candidate_labels, return_tensors="pt", padding=True) +``` + +Pass the inputs through the model, and post-process the results: + +```py +>>> import torch + +>>> with torch.no_grad(): +... outputs = model(**inputs) + +>>> logits = outputs.logits_per_image[0] +>>> probs = logits.softmax(dim=-1).numpy() +>>> scores = probs.tolist() + +>>> result = [ +... {"score": score, "label": candidate_label} +... for score, candidate_label in sorted(zip(probs, candidate_labels), key=lambda x: -x[0]) +... ] + +>>> result +[{'score': 0.998572, 'label': 'car'}, + {'score': 0.0010570387, 'label': 'bike'}, + {'score': 0.0003393686, 'label': 'tree'}, + {'score': 3.1572064e-05, 'label': 'cat'}] +``` \ No newline at end of file From 798d110324efb1116a89649e1f58cbd4a60def60 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:26:50 -0400 Subject: [PATCH 076/277] Fix doc link for MGP-STR (#22138) --- README.md | 2 +- README_es.md | 2 +- README_hd.md | 2 +- README_ja.md | 2 +- README_ko.md | 2 +- README_zh-hans.md | 2 +- README_zh-hant.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4c81448e2ab9..cd866a4ce42c 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. -1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. diff --git a/README_es.md b/README_es.md index aa83571ca0ce..3027a03caca7 100644 --- a/README_es.md +++ b/README_es.md @@ -365,7 +365,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. -1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. diff --git a/README_hd.md b/README_hd.md index ff49b28426d4..fc5fdd882c66 100644 --- a/README_hd.md +++ b/README_hd.md @@ -337,7 +337,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (फेसबुक से) साथ में पेपर [एक्स्टेंसिबल बहुभाषी प्रीट्रेनिंग और फाइनट्यूनिंग के साथ बहुभाषी अनुवाद](https://arxiv युकिंग टैंग, चाउ ट्रान, जियान ली, पेंग-जेन चेन, नमन गोयल, विश्रव चौधरी, जियाताओ गु, एंजेला फैन द्वारा .org/abs/2008.00401)। 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA से) कागज के साथ [Megatron-LM: मॉडल का उपयोग करके बहु-अरब पैरामीटर भाषा मॉडल का प्रशिक्षण Parallelism](https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा। 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA से) साथ वाला पेपर [Megatron-LM: ट्रेनिंग मल्टी-बिलियन पैरामीटर लैंग्वेज मॉडल्स यूजिंग मॉडल पैरेललिज़्म] (https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा पोस्ट किया गया। -1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research से) Peng Wang, Cheng Da, and Cong Yao. द्वाराअनुसंधान पत्र [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) के साथ जारी किया गया +1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (Alibaba Research से) Peng Wang, Cheng Da, and Cong Yao. द्वाराअनुसंधान पत्र [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) के साथ जारी किया गया 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (फ्रॉम Studio Ousia) साथ में पेपर [mLUKE: द पावर ऑफ एंटिटी रिप्रेजेंटेशन इन मल्टीलिंगुअल प्रीट्रेन्ड लैंग्वेज मॉडल्स](https://arxiv.org/abs/2110.08151) रयोकन री, इकुया यामाडा, और योशिमासा त्सुरोका द्वारा। 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (सीएमयू/गूगल ब्रेन से) साथ में कागज [मोबाइलबर्ट: संसाधन-सीमित उपकरणों के लिए एक कॉम्पैक्ट टास्क-अज्ञेय बीईआरटी] (https://arxiv.org/abs/2004.02984) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, और Denny Zhou द्वारा पोस्ट किया गया। 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. diff --git a/README_ja.md b/README_ja.md index 3803a6a2c5ee..3fae87a294fc 100644 --- a/README_ja.md +++ b/README_ja.md @@ -399,7 +399,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook から) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan から公開された研究論文: [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) -1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research から) Peng Wang, Cheng Da, and Cong Yao. から公開された研究論文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) +1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (Alibaba Research から) Peng Wang, Cheng Da, and Cong Yao. から公開された研究論文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (Studio Ousia から) Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka から公開された研究論文: [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (CMU/Google Brain から) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou から公開された研究論文: [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (Google Inc. から) Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam から公開された研究論文: [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) diff --git a/README_ko.md b/README_ko.md index 18056a12a62e..c19fabca5c90 100644 --- a/README_ko.md +++ b/README_ko.md @@ -314,7 +314,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook 에서) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 의 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 논문과 함께 발표했습니다. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. -1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research 에서 제공)은 Peng Wang, Cheng Da, and Cong Yao.의 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592)논문과 함께 발표했습니다. +1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (Alibaba Research 에서 제공)은 Peng Wang, Cheng Da, and Cong Yao.의 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592)논문과 함께 발표했습니다. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (Studio Ousia 에서) Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka 의 [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 논문과 함께 발표했습니다. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (CMU/Google Brain 에서) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou 의 [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 논문과 함께 발표했습니다. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (Google Inc. 에서) Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam 의 [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) 논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index 54fb9346b2de..f077fbca40b8 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -338,7 +338,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (来自 Facebook) 伴随论文 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 由 Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 发布。 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 -1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (来自 Alibaba Research) 伴随论文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 由 Peng Wang, Cheng Da, and Cong Yao 发布。 +1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (来自 Alibaba Research) 伴随论文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 由 Peng Wang, Cheng Da, and Cong Yao 发布。 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (来自 Studio Ousia) 伴随论文 [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 由 Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka 发布。 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (来自 CMU/Google Brain) 伴随论文 [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 由 Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou 发布。 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (来自 Google Inc.) 伴随论文 [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) 由 Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index 08ce35cbfff5..cc19893bc9ee 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -350,7 +350,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. -1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. From 729251a76728e5d096977c2fd965e7638aebd03c Mon Sep 17 00:00:00 2001 From: mollerup23 <69806327+mollerup23@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:58:29 -0400 Subject: [PATCH 077/277] Adding Type Hints to TF_Pegasus model (#21941) * Adding Type Hints to TF_Pegasus model * Updated some parameters per maintainer comments --- .../models/pegasus/modeling_tf_pegasus.py | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/transformers/models/pegasus/modeling_tf_pegasus.py b/src/transformers/models/pegasus/modeling_tf_pegasus.py index f38dbde5f37a..5955d50d61c7 100644 --- a/src/transformers/models/pegasus/modeling_tf_pegasus.py +++ b/src/transformers/models/pegasus/modeling_tf_pegasus.py @@ -712,14 +712,14 @@ def set_embed_tokens(self, embed_tokens): @unpack_inputs def call( self, - input_ids=None, - inputs_embeds=None, - attention_mask=None, - head_mask=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[tf.Tensor] = None, + inputs_embeds: Optional[tf.Tensor] = None, + attention_mask: Optional[tf.Tensor] = None, + head_mask: Optional[tf.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, ): """ Args: @@ -886,20 +886,20 @@ def set_embed_tokens(self, embed_tokens): @unpack_inputs def call( self, - input_ids=None, - inputs_embeds=None, - attention_mask=None, - position_ids=None, - encoder_hidden_states=None, - encoder_attention_mask=None, - head_mask=None, - cross_attn_head_mask=None, - past_key_values=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + input_ids: Optional[tf.Tensor] = None, + inputs_embeds: Optional[tf.Tensor] = None, + attention_mask: Optional[tf.Tensor] = None, + position_ids: Optional[tf.Tensor] = None, + encoder_hidden_states: Optional[tf.Tensor] = None, + encoder_attention_mask: Optional[tf.Tensor] = None, + head_mask: Optional[tf.Tensor] = None, + cross_attn_head_mask: Optional[tf.Tensor] = None, + past_key_values: Tuple[Tuple[tf.Tensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, ): r""" Args: @@ -1124,23 +1124,23 @@ def set_input_embeddings(self, new_embeddings): @unpack_inputs def call( self, - input_ids=None, - attention_mask=None, - decoder_input_ids=None, - decoder_attention_mask=None, - decoder_position_ids=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, + input_ids: Optional[tf.Tensor] = None, + attention_mask: Optional[tf.Tensor] = None, + decoder_input_ids: Optional[tf.Tensor] = None, + decoder_attention_mask: Optional[tf.Tensor] = None, + decoder_position_ids: Optional[tf.Tensor] = None, + head_mask: Optional[tf.Tensor] = None, + decoder_head_mask: Optional[tf.Tensor] = None, + cross_attn_head_mask: Optional[tf.Tensor] = None, encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None, - past_key_values=None, - inputs_embeds=None, - decoder_inputs_embeds=None, - use_cache=None, - output_attentions=None, - output_hidden_states=None, - return_dict=None, - training=False, + past_key_values: Tuple[Tuple[tf.Tensor]] = None, + inputs_embeds: Optional[tf.Tensor] = None, + decoder_inputs_embeds: Optional[tf.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, **kwargs, ): if decoder_input_ids is None and decoder_inputs_embeds is None: From d56ad120631142203c6177749776dbe8be60f2ae Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Mon, 13 Mar 2023 19:11:19 +0100 Subject: [PATCH 078/277] Add a new script to check model testers' config (#22063) * Add script --------- Co-authored-by: ydshieh --- utils/check_model_tester.py | 63 +++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 utils/check_model_tester.py diff --git a/utils/check_model_tester.py b/utils/check_model_tester.py new file mode 100644 index 000000000000..8ace411b1a4e --- /dev/null +++ b/utils/check_model_tester.py @@ -0,0 +1,63 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import glob +import os + +from get_test_info import get_tester_classes + + +if __name__ == "__main__": + failures = [] + + pattern = os.path.join("tests", "models", "**", "test_modeling_*.py") + test_files = glob.glob(pattern) + # TODO: deal with TF/Flax too + test_files = [ + x for x in test_files if not (x.startswith("test_modeling_tf_") or x.startswith("test_modeling_flax_")) + ] + + for test_file in test_files: + tester_classes = get_tester_classes(test_file) + for tester_class in tester_classes: + # A few tester classes don't have `parent` parameter in `__init__`. + # TODO: deal this better + try: + tester = tester_class(parent=None) + except Exception: + continue + if hasattr(tester, "get_config"): + config = tester.get_config() + for k, v in config.to_dict().items(): + if isinstance(v, int): + target = None + if k in ["vocab_size"]: + target = 100 + elif k in ["max_position_embeddings"]: + target = 128 + elif k in ["hidden_size", "d_model"]: + target = 40 + elif k == ["num_layers", "num_hidden_layers", "num_encoder_layers", "num_decoder_layers"]: + target = 5 + if target is not None and v > target: + failures.append( + f"{tester_class.__name__} will produce a `config` of type `{config.__class__.__name__}`" + f' with config["{k}"] = {v} which is too large for testing! Set its value to be smaller' + f" than {target}." + ) + + if len(failures) > 0: + raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures)) From c53441c4282f6bee1b13bad322c9f610930da4b4 Mon Sep 17 00:00:00 2001 From: bishmdl76 <68867214+bishmdl76@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:12:12 -0400 Subject: [PATCH 079/277] Update configuration_align.py (projected_dim=640) (#22139) Update configuration_align.py updated projected_dim=640 from 512 in arguments of AlignConfig --- src/transformers/models/align/configuration_align.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/align/configuration_align.py b/src/transformers/models/align/configuration_align.py index cfe1115f616f..488b7f6fe458 100644 --- a/src/transformers/models/align/configuration_align.py +++ b/src/transformers/models/align/configuration_align.py @@ -310,7 +310,7 @@ class AlignConfig(PretrainedConfig): Dictionary of configuration options used to initialize [`AlignTextConfig`]. vision_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`AlignVisionConfig`]. - projection_dim (`int`, *optional*, defaults to 512): + projection_dim (`int`, *optional*, defaults to 640): Dimentionality of text and vision projection layers. temperature_init_value (`float`, *optional*, defaults to 1.0): The inital value of the *temperature* paramter. Default is used as per the original ALIGN implementation. From f3067ccc92b04182390c65fc7ff6a3861b03ce40 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 13 Mar 2023 19:46:01 +0100 Subject: [PATCH 080/277] [`Whiper`] add `get_input_embeddings` to `WhisperForAudioClassification` (#22133) * add `get_input_embeddings` to `WhisperForAudioClassification` * add common tests * fix another common test * Update tests/models/whisper/test_modeling_whisper.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * fix style --------- Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> --- .../models/whisper/modeling_whisper.py | 20 ++++++++++++- tests/models/whisper/test_modeling_whisper.py | 30 ++++++++++++++++--- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index 94d0be40476d..e29802d444eb 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -767,6 +767,12 @@ def _freeze_parameters(self): param.requires_grad = False self._requires_grad = False + def get_input_embeddings(self) -> nn.Module: + return self.conv1 + + def set_input_embeddings(self, value: nn.Module): + self.conv1 = value + def forward( self, input_features, @@ -1023,7 +1029,10 @@ def forward( ) # embed positions - positions = self.embed_positions(input_ids, past_key_values_length=past_key_values_length) + if input_ids is not None: + positions = self.embed_positions(input_ids, past_key_values_length=past_key_values_length) + else: + positions = self.embed_positions(inputs_embeds, past_key_values_length=past_key_values_length) hidden_states = inputs_embeds + positions hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) @@ -1330,6 +1339,9 @@ def get_output_embeddings(self): def set_output_embeddings(self, new_embeddings): self.proj_out = new_embeddings + def get_input_embeddings(self) -> nn.Module: + return self.model.get_input_embeddings() + def freeze_encoder(self): """ Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will @@ -1635,6 +1647,12 @@ def freeze_encoder(self): """ self.encoder._freeze_parameters() + def get_input_embeddings(self) -> nn.Module: + return self.encoder.get_input_embeddings() + + def set_input_embeddings(self, value: nn.Module): + self.encoder.set_input_embeddings(value) + @add_start_docstrings_to_model_forward(WHISPER_ENCODER_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) def forward( diff --git a/tests/models/whisper/test_modeling_whisper.py b/tests/models/whisper/test_modeling_whisper.py index 9d99c2d4d5e5..992f4564cf6e 100644 --- a/tests/models/whisper/test_modeling_whisper.py +++ b/tests/models/whisper/test_modeling_whisper.py @@ -357,9 +357,24 @@ def _get_input_ids_and_config(self): return config, input_ids, None, max_length - # not implemented currently def test_inputs_embeds(self): - pass + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + model.to(torch_device) + model.eval() + + inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) + + decoder_input_ids = inputs.pop("decoder_input_ids", None) + inputs.pop("decoder_attention_mask", None) + + wte = model.get_input_embeddings() + inputs["decoder_inputs_embeds"] = wte(decoder_input_ids) + + with torch.no_grad(): + model(**inputs)[0] # training is not supported yet def test_training(self): @@ -1566,9 +1581,16 @@ def test_encoder_outputs(self): self.assertTrue((outputs_embeds == outputs).all()) - # WhisperEncoder has no inputs_embeds and thus the `get_input_embeddings` fn is not implemented + # Needs to override as the encoder input embedding is a Conv1d def test_model_common_attributes(self): - pass + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + self.assertIsInstance(model.get_input_embeddings(), (torch.nn.Conv1d)) + model.set_input_embeddings(torch.nn.Conv1d(10, 10, 3)) + x = model.get_output_embeddings() + self.assertTrue(x is None or isinstance(x, torch.nn.Conv1d)) # WhisperEncoder cannot resize token embeddings since it has no tokens embeddings def test_resize_tokens_embeddings(self): From e6bbfa825a15720000a1c9ad16bb6ea5d8dac868 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Mon, 13 Mar 2023 19:00:25 +0000 Subject: [PATCH 081/277] Trainer: let generate pick its inputs (#22108) * Let generate pick its inputs * fix squad seq2seq example --- src/transformers/trainer_seq2seq.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/transformers/trainer_seq2seq.py b/src/transformers/trainer_seq2seq.py index 4a79516d265c..3f7fb8211814 100644 --- a/src/transformers/trainer_seq2seq.py +++ b/src/transformers/trainer_seq2seq.py @@ -182,23 +182,11 @@ def prediction_step( gen_kwargs["synced_gpus"] if gen_kwargs.get("synced_gpus") is not None else default_synced_gpus ) - if "attention_mask" in inputs: - gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - if "global_attention_mask" in inputs: - gen_kwargs["global_attention_mask"] = inputs.get("global_attention_mask", None) - - # prepare generation inputs - # some encoder-decoder models can have varying encoder's and thus - # varying model input names - if hasattr(self.model, "encoder") and self.model.encoder.main_input_name != self.model.main_input_name: - generation_inputs = inputs[self.model.encoder.main_input_name] - else: - generation_inputs = inputs[self.model.main_input_name] + # TODO (Joao): the following line is needed to keep a consistent result on SQUAD. Ideally, we should not block + # users from preparing a dataset with `decoder_input_ids`. + inputs = {k: v for k, v in inputs.items() if k != "decoder_input_ids"} + generated_tokens = self.model.generate(**inputs, **gen_kwargs) - generated_tokens = self.model.generate( - generation_inputs, - **gen_kwargs, - ) # Temporary hack to ensure the generation config is not initialized for each iteration of the evaluation loop # TODO: remove this hack when the legacy code that initializes generation_config from a model config is # removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183 From 382b5ec55906e106092fb7e2c4fb669edfc9bd44 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 13 Mar 2023 15:50:50 -0400 Subject: [PATCH 082/277] Enforce same behavior as PyTorch 2.0 for older versions (#22136) --- src/transformers/trainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 344523842343..294815fe13e3 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -1811,7 +1811,7 @@ def _inner_training_loop( # _total_loss_scalar is updated everytime .item() has to be called on tr_loss and stores the sum of all losses self._total_loss_scalar = 0.0 self._globalstep_last_logged = self.state.global_step - model.zero_grad() + model.zero_grad(set_to_none=True) self.control = self.callback_handler.on_train_begin(args, self.state, self.control) @@ -1967,7 +1967,7 @@ def _inner_training_loop( if optimizer_was_run and not self.deepspeed: self.lr_scheduler.step() - model.zero_grad() + model.zero_grad(set_to_none=True) self.state.global_step += 1 self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch self.control = self.callback_handler.on_step_end(args, self.state, self.control) From 2e0898b9972df5f4215aaa40ec8bb25f68e77c5f Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 13 Mar 2023 12:51:40 -0700 Subject: [PATCH 083/277] [trainer] fix bug in grad accum with multiple epochs (#22098) * [trainer] fix bug in grad accum * comment out debug * fix one-off * rename counter --- src/transformers/trainer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 294815fe13e3..9bd7e0f3ba04 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -1831,6 +1831,7 @@ def _inner_training_loop( # AT THE VERY END! _ = list(train_dataloader.sampler) + total_batched_samples = 0 for epoch in range(epochs_trained, num_train_epochs): if isinstance(train_dataloader, DataLoader) and isinstance(train_dataloader.sampler, DistributedSampler): train_dataloader.sampler.set_epoch(epoch) @@ -1867,6 +1868,7 @@ def _inner_training_loop( step = -1 for step, inputs in enumerate(epoch_iterator): + total_batched_samples += 1 if rng_to_sync: self._load_rng_state(resume_from_checkpoint) rng_to_sync = False @@ -1887,7 +1889,7 @@ def _inner_training_loop( self.control = self.callback_handler.on_step_begin(args, self.state, self.control) if ( - ((step + 1) % args.gradient_accumulation_steps != 0) + (total_batched_samples % args.gradient_accumulation_steps != 0) and args.local_rank != -1 and args._no_sync_in_gradient_accumulation ): @@ -1913,7 +1915,7 @@ def _inner_training_loop( if self.deepspeed: self.deepspeed.step() - if (step + 1) % args.gradient_accumulation_steps == 0 or ( + if total_batched_samples % args.gradient_accumulation_steps == 0 or ( # last step in epoch but step is always smaller than gradient_accumulation_steps steps_in_epoch <= args.gradient_accumulation_steps and (step + 1) == steps_in_epoch From a3cb68272f782cd488e0256c87cfcf4b9727e4a6 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 13 Mar 2023 12:52:42 -0700 Subject: [PATCH 084/277] [deepspeed docs] Activation Checkpointing (#22099) * [deepspeed docs] Activation Checkpointing * Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update deepspeed.mdx --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- docs/source/en/main_classes/deepspeed.mdx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/source/en/main_classes/deepspeed.mdx b/docs/source/en/main_classes/deepspeed.mdx index 1a5358c807c7..3ee96f0b0413 100644 --- a/docs/source/en/main_classes/deepspeed.mdx +++ b/docs/source/en/main_classes/deepspeed.mdx @@ -1198,6 +1198,20 @@ Other quick related performance notes: - if you are training something from scratch always try to have tensors with shapes that are divisible by 16 (e.g. hidden size). For batch size try divisible by 2 at least. There are [wave and tile quanitization](https://developer.nvidia.com/blog/optimizing-gpu-performance-tensor-cores/) divisibility that is hardware-specific if you want to squeeze even higher performance from your GPUs. +### Activation Checkpointing or Gradient Checkpointing + +Activation checkpointing and gradient checkpointing are two distinct terms that refer to the same methodology. It's very confusing but this is how it is. + +Gradient checkpointing allows one to trade speed for GPU memory, which either allows one to overcome a GPU OOM, or increase their batch size, which often leads to a better performance. + +HF Transformers models don't know anything about DeepSpeed's activation checkpointing, so if you try to enable that feature in the DeepSpeed config file, nothing will happen. + +Therefore you have two ways to take advantage of this very beneficial feature: + +1. If you want to use a HF Transformers models you can do `model.gradient_checkpointing_enable()` or use `--gradient_checkpointing` in the HF Trainer, which will automatically enable this for you. `torch.utils.checkpoint` is used there. +2. If you write your own model and you want to use DeepSpeed's activation checkpointing you can use the [API prescribed there](https://deepspeed.readthedocs.io/en/latest/activation-checkpointing.html). You can also take the HF Transformers modeling code and replace `torch.utils.checkpoint` with the DeepSpeed's API. The latter is more flexible since it allows you to offload the forward activations to the CPU memory instead of recalculating them. + + ### Optimizer and Scheduler As long as you don't enable `offload_optimizer` you can mix and match DeepSpeed and HuggingFace schedulers and From 5a1f4f836cc00c657b43b291f7401702562ce513 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 13 Mar 2023 16:34:00 -0400 Subject: [PATCH 085/277] Remove backend check for torch.compile (#22140) * Remove backend enforcment for torch.compile * Update error * Update src/transformers/training_args.py Co-authored-by: Stas Bekman * Apply suggestions from code review Co-authored-by: Stas Bekman * Style --------- Co-authored-by: Stas Bekman --- src/transformers/trainer.py | 2 +- src/transformers/training_args.py | 34 +++++++++----------------- src/transformers/utils/import_utils.py | 2 ++ 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 9bd7e0f3ba04..a5aa54fe65b2 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -672,7 +672,7 @@ def __init__( # torch.compile if args.torch_compile and not is_torch_compile_available(): - raise RuntimeError("Using torch.compile requires a nightly install of PyTorch.") + raise RuntimeError("Using torch.compile requires PyTorch 2.0 or higher.") def add_callback(self, callback): """ diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index 32b1f0059063..acce46069dc5 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -85,20 +85,6 @@ trainer_log_levels = dict(**log_levels, passive=-1) -TORCH_COMPILE_BACKENDS = [ - "eager", - "aot_eager", - "inductor", - "nvfuser", - "aot_nvfuser", - "aot_cudagraphs", - "ofi", - "fx2trt", - "onnxrt", - "ipex", -] - - def default_logdir() -> str: """ Same default as PyTorch @@ -571,17 +557,24 @@ class TrainingArguments: Whether or not to compile the model using PyTorch 2.0 [`torch.compile`](https://pytorch.org/get-started/pytorch-2.0/) (requires a nighlty install of PyTorch). - If set, the backend will default to `"inductor"` (can be customized with `torch_compile_backend`) and the - mode will default to `"default"` (can be customized with `torch_compile_mode`). + This will use the best defaults for the [`torch.compile` + API](https://pytorch.org/docs/2.0/generated/torch.compile.html?highlight=torch+compile#torch.compile). You + can customize the defaults with the argument `torch_compile_backend` and `torch_compile_mode` but we don't + guarantee any of them will work as the support is progressively rolled in in PyTorch. + + This flag and the whole compile API is experimental and subject to change in future releases. torch_compile_backend (`str`, *optional*): The backend to use in `torch.compile`. If set to any value, `torch_compile` will be set to `True`. - Possible choices are `"eager"`, `"aot_eager"`, `"inductor"`, `"nvfuser"`, `"aot_nvfuser"`, - `"aot_cudagraphs"`, `"ofi"`, `"fx2trt"`, `"onnxrt"` and `"ipex"`. + Refer to the PyTorch doc for possible values and note that they may change across PyTorch versions. + + This flag is experimental and subject to change in future releases. torch_compile_mode (`str`, *optional*): The mode to use in `torch.compile`. If set to any value, `torch_compile` will be set to `True`. - Possible choices are `"default"`, `"reduce-overhead"` and `"max-autotune"`. + Refer to the PyTorch doc for possible values and note that they may change across PyTorch versions. + + This flag is experimental and subject to change in future releases. """ framework = "pt" @@ -1061,7 +1054,6 @@ class TrainingArguments: default=None, metadata={ "help": "This argument is deprecated, use `--torch_compile_backend` instead.", - "choices": TORCH_COMPILE_BACKENDS, }, ) ray_scope: Optional[str] = field( @@ -1090,14 +1082,12 @@ class TrainingArguments: default=None, metadata={ "help": "Which backend to use with `torch.compile`, passing one will trigger a model compilation.", - "choices": TORCH_COMPILE_BACKENDS, }, ) torch_compile_mode: Optional[str] = field( default=None, metadata={ "help": "Which mode to use with `torch.compile`, passing one will trigger a model compilation.", - "choices": ["default", "reduce-overhead", "max-autotune"], }, ) diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index 6ef639831b8d..cfa7f44d9849 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -478,6 +478,8 @@ def is_torch_compile_available(): import torch + # We don't do any version check here to support nighlies marked as 1.14. Ultimately needs to check version against + # 2.0 but let's do it later. return hasattr(torch, "compile") From 7703f02f4a71500f206c0f65b399a68f358f4a64 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 13 Mar 2023 21:39:06 +0100 Subject: [PATCH 086/277] [Safetensors] Add explicit flag to from pretrained (#22083) * [Safetensors] Add explicit flag to from pretrained * add test * remove @ * Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/modeling_utils.py | 11 ++++++-- tests/test_modeling_common.py | 45 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e85d8e304d09..6a76accf5691 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2086,6 +2086,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P subfolder = kwargs.pop("subfolder", "") commit_hash = kwargs.pop("_commit_hash", None) variant = kwargs.pop("variant", None) + use_safetensors = kwargs.pop("use_safetensors", None if is_safetensors_available() else False) if trust_remote_code is True: logger.warning( @@ -2222,14 +2223,14 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P ): # Load from a Flax checkpoint in priority if from_flax archive_file = os.path.join(pretrained_model_name_or_path, subfolder, FLAX_WEIGHTS_NAME) - elif is_safetensors_available() and os.path.isfile( + elif use_safetensors is not False and os.path.isfile( os.path.join(pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant)) ): # Load from a safetensors checkpoint archive_file = os.path.join( pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_NAME, variant) ) - elif is_safetensors_available() and os.path.isfile( + elif use_safetensors is not False and os.path.isfile( os.path.join( pretrained_model_name_or_path, subfolder, _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant) ) @@ -2295,7 +2296,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P filename = TF2_WEIGHTS_NAME elif from_flax: filename = FLAX_WEIGHTS_NAME - elif is_safetensors_available(): + elif use_safetensors is not False: filename = _add_variant(SAFE_WEIGHTS_NAME, variant) else: filename = _add_variant(WEIGHTS_NAME, variant) @@ -2328,6 +2329,10 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P ) if resolved_archive_file is not None: is_sharded = True + elif use_safetensors: + raise EnvironmentError( + f" {_add_variant(SAFE_WEIGHTS_NAME, variant)} or {_add_variant(SAFE_WEIGHTS_INDEX_NAME, variant)} and thus cannot be loaded with `safetensors`. Please make sure that the model has been saved with `safe_serialization=True` or do not set `use_safetensors=True`." + ) else: # This repo has no safetensors file of any kind, we switch to PyTorch. filename = _add_variant(WEIGHTS_NAME, variant) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index cb06400e9a77..f71366d21838 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -15,6 +15,7 @@ import copy import gc +import glob import inspect import json import os @@ -119,6 +120,7 @@ AutoTokenizer, BertConfig, BertModel, + CLIPTextModel, PreTrainedModel, T5Config, T5ForConditionalGeneration, @@ -3327,6 +3329,49 @@ def test_legacy_load_from_url(self): "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/pytorch_model.bin", config=config ) + @require_safetensors + def test_use_safetensors(self): + # test nice error message if no safetensor files available + with self.assertRaises(OSError) as env_error: + AutoModel.from_pretrained("hf-internal-testing/tiny-random-RobertaModel", use_safetensors=True) + + self.assertTrue( + "model.safetensors or model.safetensors.index.json and thus cannot be loaded with `safetensors`" + in str(env_error.exception) + ) + + # test that error if only safetensors is available + with self.assertRaises(OSError) as env_error: + BertModel.from_pretrained("hf-internal-testing/tiny-random-bert-safetensors", use_safetensors=False) + + self.assertTrue("does not appear to have a file named pytorch_model.bin" in str(env_error.exception)) + + # test that only safetensors if both available and use_safetensors=False + with tempfile.TemporaryDirectory() as tmp_dir: + CLIPTextModel.from_pretrained( + "hf-internal-testing/diffusers-stable-diffusion-tiny-all", + subfolder="text_encoder", + use_safetensors=False, + cache_dir=tmp_dir, + ) + + all_downloaded_files = glob.glob(os.path.join(tmp_dir, "*", "snapshots", "*", "*", "*")) + self.assertTrue(any(f.endswith("bin") for f in all_downloaded_files)) + self.assertFalse(any(f.endswith("safetensors") for f in all_downloaded_files)) + + # test that no safetensors if both available and use_safetensors=True + with tempfile.TemporaryDirectory() as tmp_dir: + CLIPTextModel.from_pretrained( + "hf-internal-testing/diffusers-stable-diffusion-tiny-all", + subfolder="text_encoder", + use_safetensors=True, + cache_dir=tmp_dir, + ) + + all_downloaded_files = glob.glob(os.path.join(tmp_dir, "*", "snapshots", "*", "*", "*")) + self.assertTrue(any(f.endswith("safetensors") for f in all_downloaded_files)) + self.assertFalse(any(f.endswith("bin") for f in all_downloaded_files)) + @require_safetensors def test_safetensors_save_and_load(self): model = BertModel.from_pretrained("hf-internal-testing/tiny-random-bert") From 4c876a344923174c76ed70c0d3b700efaf2a0c14 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Mon, 13 Mar 2023 22:21:15 +0100 Subject: [PATCH 087/277] Prepare daily CI for torch 2.0.0 (#22135) Co-authored-by: ydshieh --- .github/workflows/build-docker-images.yml | 10 +++++++ docker/transformers-all-latest-gpu/Dockerfile | 26 ++++++++++--------- .../Dockerfile | 14 ++++++---- docker/transformers-pytorch-gpu/Dockerfile | 14 +++++----- 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 03ecf450264d..6627812a666b 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -22,6 +22,16 @@ jobs: name: "Latest PyTorch + TensorFlow [dev]" runs-on: ubuntu-latest steps: + - name: Cleanup disk + run: | + sudo ls -l /usr/local/lib/ + sudo ls -l /usr/share/ + sudo du -sh /usr/local/lib/ + sudo du -sh /usr/share/ + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo du -sh /usr/local/lib/ + sudo du -sh /usr/share/ - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 diff --git a/docker/transformers-all-latest-gpu/Dockerfile b/docker/transformers-all-latest-gpu/Dockerfile index 46d8b127b8e5..9f30f4531b0a 100644 --- a/docker/transformers-all-latest-gpu/Dockerfile +++ b/docker/transformers-all-latest-gpu/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04 +FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu20.04 LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive @@ -9,11 +9,11 @@ SHELL ["sh", "-lc"] # The following `ARG` are mainly used to specify the versions explicitly & directly in this docker file, and not meant # to be used as arguments for docker build (so far). -ARG PYTORCH='1.13.1' +ARG PYTORCH='2.0.0' # (not always a valid torch version) ARG INTEL_TORCH_EXT='1.11.0' # Example: `cu102`, `cu113`, etc. -ARG CUDA='cu116' +ARG CUDA='cu117' RUN apt update RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs @@ -24,13 +24,15 @@ ARG REF=main RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] -# TODO: Handle these in a python utility script -RUN [ ${#PYTORCH} -gt 0 -a "$PYTORCH" != "pre" ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; echo "export VERSION='$VERSION'" >> ~/.profile -RUN echo torch=$VERSION -# `torchvision` and `torchaudio` should be installed along with `torch`, especially for nightly build. -# Currently, let's just use their latest releases (when `torch` is installed with a release version) -# TODO: We might need to specify proper versions that work with a specific torch version (especially for past CI). -RUN [ "$PYTORCH" != "pre" ] && python3 -m pip install --no-cache-dir -U $VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA || python3 -m pip install --no-cache-dir -U --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/$CUDA +## TODO: Handle these in a python utility script +#RUN [ ${#PYTORCH} -gt 0 -a "$PYTORCH" != "pre" ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; echo "export VERSION='$VERSION'" >> ~/.profile +#RUN echo torch=$VERSION +## `torchvision` and `torchaudio` should be installed along with `torch`, especially for nightly build. +## Currently, let's just use their latest releases (when `torch` is installed with a release version) +## TODO: We might need to specify proper versions that work with a specific torch version (especially for past CI). +#RUN [ "$PYTORCH" != "pre" ] && python3 -m pip install --no-cache-dir -U $VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA || python3 -m pip install --no-cache-dir -U --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/$CUDA + +RUN python3 -m pip install --no-cache-dir -U torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 RUN python3 -m pip install --no-cache-dir -U tensorflow==2.11 RUN python3 -m pip install --no-cache-dir -U tensorflow_probability @@ -54,8 +56,8 @@ RUN python3 -m pip install --no-cache-dir bitsandbytes # For video model testing RUN python3 -m pip install --no-cache-dir decord av==9.2.0 -# For `dinat` model -RUN python3 -m pip install --no-cache-dir natten -f https://shi-labs.com/natten/wheels/$CUDA/ +## For `dinat` model +#RUN python3 -m pip install --no-cache-dir natten -f https://shi-labs.com/natten/wheels/$CUDA/ # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile index 57ddc43506aa..1c20ee261be3 100644 --- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile +++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile @@ -1,12 +1,12 @@ -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel_22-04.html#rel_22-04 -FROM nvcr.io/nvidia/pytorch:22.04-py3 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel_22-08.html#rel_22-08 +FROM nvcr.io/nvidia/pytorch:22.08-py3 LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive ARG PYTORCH='1.13.1' # Example: `cu102`, `cu113`, etc. -ARG CUDA='cu116' +ARG CUDA='cu117' RUN apt -y update RUN apt install -y libaio-dev @@ -18,11 +18,15 @@ RUN git clone https://github.com/huggingface/transformers && cd transformers && # Install latest release PyTorch # (PyTorch must be installed before pre-compiling any DeepSpeed c++/cuda ops.) # (https://www.deepspeed.ai/tutorials/advanced-install/#pre-install-deepspeed-ops) -RUN python3 -m pip install --no-cache-dir -U torch==$PYTORCH torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA +#RUN python3 -m pip install --no-cache-dir -U torch==$PYTORCH torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA +RUN python3 -m pip install --no-cache-dir -U torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 + RUN python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing] -RUN python3 -m pip install torch-tensorrt==1.3.0 --find-links https://github.com/pytorch/TensorRT/releases/expanded_assets/v1.3.0 +# This will uninstall torch 2.0.0 +# TODO: uncomment the following line once `torch-tensorrt` is ready for `torch 2.0.0` +# RUN python3 -m pip install torch-tensorrt==1.3.0 --find-links https://github.com/pytorch/TensorRT/releases/expanded_assets/v1.3.0 # recompile apex RUN python3 -m pip uninstall -y apex diff --git a/docker/transformers-pytorch-gpu/Dockerfile b/docker/transformers-pytorch-gpu/Dockerfile index 689c18435bb4..24564aca6fae 100644 --- a/docker/transformers-pytorch-gpu/Dockerfile +++ b/docker/transformers-pytorch-gpu/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04 +FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu20.04 LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive @@ -12,15 +12,17 @@ RUN git clone https://github.com/huggingface/transformers && cd transformers && RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-torch,testing,video] # If set to nothing, will install the latest version -ARG PYTORCH='1.13.1' +ARG PYTORCH='2.0.0' ARG TORCH_VISION='' ARG TORCH_AUDIO='' # Example: `cu102`, `cu113`, etc. -ARG CUDA='cu116' +ARG CUDA='cu117' -RUN [ ${#PYTORCH} -gt 0 ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA -RUN [ ${#TORCH_VISION} -gt 0 ] && VERSION='torchvision=='TORCH_VISION'.*' || VERSION='torchvision'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA -RUN [ ${#TORCH_AUDIO} -gt 0 ] && VERSION='torchaudio=='TORCH_AUDIO'.*' || VERSION='torchaudio'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA +#RUN [ ${#PYTORCH} -gt 0 ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA +#RUN [ ${#TORCH_VISION} -gt 0 ] && VERSION='torchvision=='TORCH_VISION'.*' || VERSION='torchvision'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA +#RUN [ ${#TORCH_AUDIO} -gt 0 ] && VERSION='torchaudio=='TORCH_AUDIO'.*' || VERSION='torchaudio'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA + +RUN python3 -m pip install --no-cache-dir -U torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 RUN python3 -m pip uninstall -y tensorflow flax From f3d55b5277aac5352cefdc97e4b528bd98b68f3d Mon Sep 17 00:00:00 2001 From: MichaelRipa <51883134+MichaelRipa@users.noreply.github.com> Date: Mon, 13 Mar 2023 19:09:37 -0400 Subject: [PATCH 088/277] docs: New terms and updates to glossary (#21982) * Updated glossary with new terms, added abbreviations for certain terms and merged autoencoding models, autoregressive models and causal language modeling into encoder and decoder models * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Added link to 'Pipeline for inference' tutorial * Trigger CI * Update docs/source/en/glossary.mdx Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Added entry for self supervised learning, added deleted entries + fixed broken links * Update docs/source/en/glossary.mdx Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --------- Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- docs/source/en/glossary.mdx | 104 ++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/docs/source/en/glossary.mdx b/docs/source/en/glossary.mdx index 4c984f389b92..6d47f647642b 100644 --- a/docs/source/en/glossary.mdx +++ b/docs/source/en/glossary.mdx @@ -73,13 +73,13 @@ by the tokenizer under the key "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] ``` -### autoencoding models +### autoencoding models -see [masked language modeling](#masked-language-modeling) +See [encoder models](#encoder-models) and [masked language modeling](#masked-language-modeling-mlm) ### autoregressive models -see [causal language modeling](#causal-language-modeling) +See [causal language modeling](#causal-language-modeling) and [decoder models](#decoder-models) ## B @@ -89,15 +89,15 @@ The backbone is the network (embeddings and layers) that outputs the raw hidden ## C -### channel - -Color images are made up of some combination of values in three channels - red, green, and blue (RGB) - and grayscale images only have one channel. In 🤗 Transformers, the channel can be the first or last dimension of an image's tensor: [`n_channels`, `height`, `width`] or [`height`, `width`, `n_channels`]. - ### causal language modeling A pretraining task where the model reads the texts in order and has to predict the next word. It's usually done by reading the whole sentence but using a mask inside the model to hide the future tokens at a certain timestep. +### channel + +Color images are made up of some combination of values in three channels - red, green, and blue (RGB) - and grayscale images only have one channel. In 🤗 Transformers, the channel can be the first or last dimension of an image's tensor: [`n_channels`, `height`, `width`] or [`height`, `width`, `n_channels`]. + ### connectionist temporal classification (CTC) An algorithm which allows a model to learn without knowing exactly how the input and output are aligned; CTC calculates the distribution of all possible outputs for a given input and chooses the most likely output from it. CTC is commonly used in speech recognition tasks because speech doesn't always cleanly align with the transcript for a variety of reasons such as a speaker's different speech rates. @@ -119,12 +119,31 @@ passing the `labels` is the preferred way to handle training. Please check each model's docs to see how they handle these input IDs for sequence to sequence training. -### deep learning +### decoder models + +Also referred to as autoregressive models, decoder models involve a pretraining task (called causal language modeling) where the model reads the texts in order and has to predict the next word. It's usually done by +reading the whole sentence with a mask to hide future tokens at a certain timestep. + + + +### deep learning (DL) Machine learning algorithms which uses neural networks with several layers. +## E + +### encoder models + +Also known as autoencoding models, encoder models take an input (such as text or images) and transform them into a condensed numerical representation called an embedding. Oftentimes, encoder models are pretrained using techniques like [masked language modeling](#masked-language-modeling-mlm), which masks parts of the input sequence and forces the model to create more meaningful representations. + + + ## F +### feature extraction + +The process of selecting and transforming raw data into a set of features that are more informative and useful for machine learning algorithms. Some examples of feature extraction include transforming raw text into word embeddings and extracting important features such as edges or shapes from image/video data. + ### feed forward chunking In each residual attention block in transformers the self-attention layer is usually followed by 2 feed forward layers. @@ -144,6 +163,12 @@ For models employing the function [`apply_chunking_to_forward`], the `chunk_size embeddings that are computed in parallel and thus defines the trade-off between memory and time complexity. If `chunk_size` is set to 0, no feed forward chunking is done. +### finetuned models + +Finetuning is a form of transfer learning which involves taking a pretrained model, freezing its weights, and replacing the output layer with a newly added [model head](#head). The model head is trained on your target dataset. + +See the [Fine-tune a pretrained model](https://huggingface.co/docs/transformers/training) tutorial for more details, and learn how to fine-tune models with 🤗 Transformers. + ## H ### head @@ -160,6 +185,10 @@ The model head refers to the last layer of a neural network that accepts the raw Vision-based Transformers models split an image into smaller patches which are linearly embedded, and then passed as a sequence to the model. You can find the `patch_size` - or resolution - of the model in it's configuration. +### inference + +Inference is the process of evaluating a model on new data after training is complete. See the [Pipeline for inference](https://huggingface.co/docs/transformers/pipeline_tutorial) tutorial to learn how to perform inference with 🤗 Transformers. + ### input IDs The input ids are often the only required parameters to be passed to the model as input. They are token indices, @@ -269,9 +298,13 @@ about their specific labels! The base models ([`BertModel`]) do not accept labels, as these are the base transformer models, simply outputting features. +### large language models (LLM) + +A generic term that refers to transformer language models (GPT-3, BLOOM, OPT) that were trained on a large quantity of data. These models also tend to have a large number of learnable parameters (e.g. 175 billion for GPT-3). + ## M -### masked language modeling +### masked language modeling (MLM) A pretraining task where the model sees a corrupted version of the texts, usually done by masking some tokens randomly, and has to predict the original text. @@ -282,21 +315,27 @@ A task that combines texts with another kind of inputs (for instance images). ## N -### Natural language generation +### Natural language generation (NLG) -All tasks related to generating text (for instance talk with transformers, translation). +All tasks related to generating text (for instance, [Write With Transformers](https://transformer.huggingface.co/), translation). -### Natural language processing +### Natural language processing (NLP) A generic way to say "deal with texts". -### Natural language understanding +### Natural language understanding (NLU) All tasks related to understanding what is in a text (for instance classifying the whole text, individual words). ## P +### pipeline + +A pipeline in 🤗 Transformers is an abstraction referring to a series of steps that are executed in a specific order to preprocess and transform data and return a prediction from a model. Some example stages found in a pipeline might be data preprocessing, feature extraction, and normalization. + +For more details, see [Pipelines for inference](https://huggingface.co/docs/transformers/pipeline_tutorial). + ### pixel values A tensor of the numerical representations of an image that is passed to a model. The pixel values have a shape of [`batch_size`, `num_channels`, `height`, `width`], and are generated from an image processor. @@ -317,22 +356,29 @@ absolute positional embeddings. Absolute positional embeddings are selected in the range `[0, config.max_position_embeddings - 1]`. Some models use other types of positional embeddings, such as sinusoidal position embeddings or relative position embeddings. +### preprocessing + +The task of preparing raw data into a format that can be easily consumed by machine learning models. For example, text is typically preprocessed by tokenization. To gain a better idea of what preprocessing looks like for other input types, check out the [Preprocess](https://huggingface.co/docs/transformers/preprocessing) tutorial. ### pretrained model A model that has been pretrained on some data (for instance all of Wikipedia). Pretraining methods involve a self-supervised objective, which can be reading the text and trying to predict the next word (see [causal language modeling](#causal-language-modeling)) or masking some words and trying to predict them (see [masked language -modeling](#masked-language-modeling)). +modeling](#masked-language-modeling-mlm)). Speech and vision models have their own pretraining objectives. For example, Wav2Vec2 is a speech model pretrained on a contrastive task which requires the model to identify the "true" speech representation from a set of "false" speech representations. On the other hand, BEiT is a vision model pretrained on a masked image modeling task which masks some of the image patches and requires the model to predict the masked patches (similar to the masked language modeling objective). ## R -### recurrent neural network +### recurrent neural network (RNN) A type of model that uses a loop over a layer to process texts. +### representation learning + +A subfield of machine learning which focuses on learning meaningful representations of raw data. Some examples of representation learning techniques include word embeddings, autoencoders, and Generative Adversarial Networks (GANs). + ## S ### sampling rate @@ -343,6 +389,18 @@ A measurement in hertz of the number of samples (the audio signal) taken per sec Each element of the input finds out which other elements of the input they should attend to. +### self-supervised learning + +A category of machine learning techniques in which a model creates its own learning objective from unlabeled data. It differs from [unsupervised learning](#unsupervised-learning) and [supervised learning](#supervised-learning) in that the learning process is supervised, but not explicitly from the user. + +One example of self-supervised learning is [masked language modeling](#masked-language-modeling-mlm), where a model is passed sentences with a proportion of its tokens removed and learns to predict the missing tokens. + +### semi-supervised learning + +A broad category of machine learning training techniques that leverages a small amount of labeled data with a larger quantity of unlabeled data to improve the accuracy of a model, unlike [supervised learning](#supervised-learning) and [unsupervised learning](#unsupervised-learning). + +An example of a semi-supervised learning approach is "self-training", in which a model is trained on labeled data, and then used to make predictions on the unlabeled data. The portion of the unlabeled data that the model predicts with the most confidence gets added to the labeled dataset and used to retrain the model. + ### sequence-to-sequence (seq2seq) Models that generate a new sequence from an input, like translation models, or summarization models (such as @@ -352,6 +410,10 @@ Models that generate a new sequence from an input, like translation models, or s In [convolution](#convolution) or [pooling](#pooling), the stride refers to the distance the kernel is moved over a matrix. A stride of 1 means the kernel is moved one pixel over at a time, and a stride of 2 means the kernel is moved two pixels over at a time. +### supervised learning + +A form of model training that directly uses labeled data to correct and instruct model performance. Data is fed into the model being trained, and its predictions are compared to the known labels. The model updates its weights based on how incorrect its predictions were, and the process is repeated to optimize model performance. + ## T ### token @@ -410,6 +472,16 @@ sequence, corresponding to the "question", has all its tokens represented by a ` Some models, like [`XLNetModel`] use an additional token represented by a `2`. +### transfer learning + +A technique that involves taking a pretrained model and adapting it to a dataset specific to your task. Instead of training a model from scratch, you can leverage knowledge obtained from an existing model as a starting point. This speeds up the learning process and reduces the amount of training data needed. + ### transformer -Self-attention based deep learning model architecture. \ No newline at end of file +Self-attention based deep learning model architecture. + +## U + +### unsupervised learning + +A form of model training in which data provided to the model is not labeled. Unsupervised learning techniques leverage statistical information of the data distribution to find patterns useful for the task at hand. From 5c055e41425077dab9c45f5924c32b06bbc1012a Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Tue, 14 Mar 2023 09:23:48 +0100 Subject: [PATCH 089/277] =?UTF-8?q?[=F0=9F=9B=A0=EF=B8=8F]=20Fix-whisper-b?= =?UTF-8?q?reaking-changes=20=20(#21965)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * temp fix * temporary fix * update * fix tests * fixup * update based on reveiew Co-authored-by: Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> * update to fix tests * update docstring --------- Co-authored-by: Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> --- .../models/whisper/modeling_whisper.py | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index e29802d444eb..d6ec39168cd1 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -14,7 +14,6 @@ # limitations under the License. """ PyTorch Whisper model.""" - import math import random from typing import Optional, Tuple, Union @@ -37,6 +36,7 @@ from ...modeling_utils import PreTrainedModel from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings from .configuration_whisper import WhisperConfig +from .tokenization_whisper import TASK_IDS, TO_LANGUAGE_CODE logger = logging.get_logger(__name__) @@ -1510,8 +1510,8 @@ def generate( Task to use for generation, either "translate" or "transcribe". The `model.config.forced_decoder_ids` will be updated accordingly. language (`bool`, *optional*): - Language token to use for generation, should be in the form `<|en|>`. You can find all the possible - language tokens in the `model.generation_config.lang_to_id` dictionary. + Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. You can + find all the possible language tokens in the `model.generation_config.lang_to_id` dictionary. is_multilingual (`bool`, *optional*): Whether or not the model is multilingual. kwargs: @@ -1543,39 +1543,63 @@ def generate( generation_config = self.generation_config if return_timestamps is not None: - generation_config.return_timestamps = return_timestamps - - if task is not None: - generation_config.task = task + if not hasattr(generation_config, "no_timestamps_token_id"): + raise ValueError( + "You are trying to return timestamps, but the generation config is not properly set." + "Make sure to initialize the generation config with the correct attributes that are needed such as `no_timestamps_token_id`." + "For more details on how to generate the approtiate config, refer to https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363" + ) - if is_multilingual is not None: - generation_config.is_multilingual = is_multilingual + generation_config.return_timestamps = return_timestamps + else: + generation_config.return_timestamps = False if language is not None: generation_config.language = language + if task is not None: + generation_config.task = task forced_decoder_ids = [] - - if hasattr(generation_config, "is_multilingual") and generation_config.is_multilingual: + if task is not None or language is not None: if hasattr(generation_config, "language"): - forced_decoder_ids.append((1, generation_config.lang_to_id[generation_config.language])) + if generation_config.language in generation_config.lang_to_id.keys(): + language_token = generation_config.language + elif generation_config.language in TO_LANGUAGE_CODE.keys(): + language_token = f"<|{TO_LANGUAGE_CODE[generation_config.language]}|>" + else: + raise ValueError( + f"Unsupported language: {self.language}. Language should be one of:" + f" {list(TO_LANGUAGE_CODE.keys()) if generation_config.language in TO_LANGUAGE_CODE.keys() else list(TO_LANGUAGE_CODE.values())}." + ) + forced_decoder_ids.append((1, generation_config.lang_to_id[language_token])) else: - forced_decoder_ids.append((1, None)) + forced_decoder_ids.append((1, None)) # automatically detect the language if hasattr(generation_config, "task"): - forced_decoder_ids.append((2, generation_config.task_to_id[generation_config.task])) + if generation_config.task in TASK_IDS: + forced_decoder_ids.append((2, generation_config.task_to_id[generation_config.task])) + else: + raise ValueError( + f"The `{generation_config.task}`task is not supported. The task should be one of `{TASK_IDS}`" + ) else: - forced_decoder_ids.append((2, generation_config.task_to_id["transcribe"])) - - if ( - hasattr(generation_config, "return_timestamps") and generation_config.return_timestamps - ) or return_timestamps: - logits_processor = [WhisperTimeStampLogitsProcessor(generation_config)] - else: - if forced_decoder_ids and forced_decoder_ids[-1][0] != generation_config.no_timestamps_token_id: + forced_decoder_ids.append((2, generation_config.task_to_id["transcribe"])) # defaults to transcribe + if hasattr(generation_config, "no_timestamps_token_id") and not generation_config.return_timestamps: idx = forced_decoder_ids[-1][0] + 1 if forced_decoder_ids else 1 forced_decoder_ids.append((idx, generation_config.no_timestamps_token_id)) + # Legacy code for backward compatibility + elif hasattr(self.config, "forced_decoder_ids") and self.config.forced_decoder_ids is not None: + forced_decoder_ids = self.config.forced_decoder_ids + elif ( + hasattr(self.generation_config, "forced_decoder_ids") + and self.generation_config.forced_decoder_ids is not None + ): + forced_decoder_ids = self.generation_config.forced_decoder_ids + + if generation_config.return_timestamps: + logits_processor = [WhisperTimeStampLogitsProcessor(generation_config)] + if len(forced_decoder_ids) > 0: generation_config.forced_decoder_ids = forced_decoder_ids From cc22fe6662e5c9ae5a2fd014c991a1e731fdb3f3 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 14 Mar 2023 10:03:02 +0100 Subject: [PATCH 090/277] Move `is_pipeline_test_to_skip` to specific model test classes (#21999) * Move `is_pipeline_test_to_skip` to specific model test classes --------- Co-authored-by: ydshieh --- .../commands/add_new_model_like.py | 100 +++++++++++++++- tests/models/ctrl/test_modeling_ctrl.py | 12 ++ tests/models/ctrl/test_modeling_tf_ctrl.py | 12 ++ .../models/flaubert/test_modeling_flaubert.py | 20 ++++ .../flaubert/test_modeling_tf_flaubert.py | 20 ++++ tests/models/fnet/test_modeling_fnet.py | 9 ++ tests/models/gptj/test_modeling_gptj.py | 16 +++ tests/models/gptj/test_modeling_tf_gptj.py | 16 +++ .../models/layoutlm/test_modeling_layoutlm.py | 14 +++ .../layoutlmv2/test_modeling_layoutlmv2.py | 23 ++++ .../layoutlmv3/test_modeling_layoutlmv3.py | 6 + .../layoutlmv3/test_modeling_tf_layoutlmv3.py | 6 + tests/models/lilt/test_modeling_lilt.py | 6 + .../longformer/test_modeling_longformer.py | 16 +++ .../longformer/test_modeling_tf_longformer.py | 16 +++ tests/models/m2m_100/test_modeling_m2m_100.py | 11 ++ tests/models/mvp/test_modeling_mvp.py | 16 +++ tests/models/openai/test_modeling_openai.py | 12 ++ .../models/openai/test_modeling_tf_openai.py | 12 ++ tests/models/opt/test_modeling_opt.py | 16 +++ tests/models/plbart/test_modeling_plbart.py | 11 ++ .../prophetnet/test_modeling_prophetnet.py | 12 ++ .../models/reformer/test_modeling_reformer.py | 16 +++ .../models/roc_bert/test_modeling_roc_bert.py | 18 +++ .../roformer/test_modeling_tf_roformer.py | 9 ++ tests/models/tapas/test_modeling_tapas.py | 6 + tests/models/tapas/test_modeling_tf_tapas.py | 6 + .../transfo_xl/test_modeling_tf_transfo_xl.py | 12 ++ .../transfo_xl/test_modeling_transfo_xl.py | 12 ++ tests/models/xlm/test_modeling_tf_xlm.py | 20 ++++ tests/models/xlm/test_modeling_xlm.py | 20 ++++ tests/test_pipeline_mixin.py | 113 +----------------- 32 files changed, 506 insertions(+), 108 deletions(-) diff --git a/src/transformers/commands/add_new_model_like.py b/src/transformers/commands/add_new_model_like.py index 30d273bec50e..0525ad2eb6e5 100644 --- a/src/transformers/commands/add_new_model_like.py +++ b/src/transformers/commands/add_new_model_like.py @@ -173,6 +173,56 @@ def parse_module_content(content: str) -> List[str]: return objects +def extract_block(content: str, indent_level: int = 0) -> str: + """Return the first block in `content` with the indent level `indent_level`. + + The first line in `content` should be indented at `indent_level` level, otherwise an error will be thrown. + + This method will immediately stop the search when a (non-empty) line with indent level less than `indent_level` is + encountered. + + Args: + content (`str`): The content to parse + indent_level (`int`, *optional*, default to 0): The indent level of the blocks to search for + + Returns: + `str`: The first block in `content` with the indent level `indent_level`. + """ + current_object = [] + lines = content.split("\n") + # Doc-styler takes everything between two triple quotes in docstrings, so we need a fake """ here to go with this. + end_markers = [")", "]", "}", '"""'] + + for idx, line in enumerate(lines): + if idx == 0 and indent_level > 0 and not is_empty_line(line) and find_indent(line) != indent_level: + raise ValueError( + f"When `indent_level > 0`, the first line in `content` should have indent level {indent_level}. Got " + f"{find_indent(line)} instead." + ) + + if find_indent(line) < indent_level and not is_empty_line(line): + break + + # End of an object + is_valid_object = len(current_object) > 0 + if ( + not is_empty_line(line) + and not line.endswith(":") + and find_indent(line) == indent_level + and is_valid_object + ): + # Closing parts should be included in current object + if line.lstrip() in end_markers: + current_object.append(line) + return "\n".join(current_object) + else: + current_object.append(line) + + # Add last object + if len(current_object) > 0: + return "\n".join(current_object) + + def add_content_to_text( text: str, content: str, @@ -402,12 +452,53 @@ def get_module_from_file(module_file: Union[str, os.PathLike]) -> str: _re_class_func = re.compile(r"^(?:class|def)\s+([^\s:\(]+)\s*(?:\(|\:)", flags=re.MULTILINE) +def remove_attributes(obj, target_attr): + """Remove `target_attr` in `obj`.""" + lines = obj.split(os.linesep) + + target_idx = None + for idx, line in enumerate(lines): + # search for assignment + if line.lstrip().startswith(f"{target_attr} = "): + target_idx = idx + break + # search for function/method definition + elif line.lstrip().startswith(f"def {target_attr}("): + target_idx = idx + break + + # target not found + if target_idx is None: + return obj + + line = lines[target_idx] + indent_level = find_indent(line) + # forward pass to find the ending of the block (including empty lines) + parsed = extract_block("\n".join(lines[target_idx:]), indent_level) + num_lines = len(parsed.split("\n")) + for idx in range(num_lines): + lines[target_idx + idx] = None + + # backward pass to find comments or decorator + for idx in range(target_idx - 1, -1, -1): + line = lines[idx] + if (line.lstrip().startswith("#") or line.lstrip().startswith("@")) and find_indent(line) == indent_level: + lines[idx] = None + else: + break + + new_obj = os.linesep.join([x for x in lines if x is not None]) + + return new_obj + + def duplicate_module( module_file: Union[str, os.PathLike], old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns, dest_file: Optional[str] = None, add_copied_from: bool = True, + attrs_to_remove: List[str] = None, ): """ Create a new module from an existing one and adapting all function and classes names from old patterns to new ones. @@ -491,8 +582,14 @@ def duplicate_module( new_objects.append(obj) + content = "\n".join(new_objects) + # Remove some attributes that we don't want to copy to the new file(s) + if attrs_to_remove is not None: + for attr in attrs_to_remove: + content = remove_attributes(content, target_attr=attr) + with open(dest_file, "w", encoding="utf-8") as f: - content = f.write("\n".join(new_objects)) + f.write(content) def filter_framework_files( @@ -1294,6 +1391,7 @@ def disable_fx_test(filename: Path) -> bool: new_model_patterns, dest_file=dest_file, add_copied_from=False, + attrs_to_remove=["pipeline_model_mapping", "is_pipeline_test_to_skip"], ) disabled_fx_test = disabled_fx_test | disable_fx_test(dest_file) diff --git a/tests/models/ctrl/test_modeling_ctrl.py b/tests/models/ctrl/test_modeling_ctrl.py index b42db21aa65c..87d719853ad7 100644 --- a/tests/models/ctrl/test_modeling_ctrl.py +++ b/tests/models/ctrl/test_modeling_ctrl.py @@ -210,6 +210,18 @@ class CTRLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_resize_embeddings = False test_head_masking = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny + # config could not be created. + return True + + return False + def setUp(self): self.model_tester = CTRLModelTester(self) self.config_tester = ConfigTester(self, config_class=CTRLConfig, n_embd=37) diff --git a/tests/models/ctrl/test_modeling_tf_ctrl.py b/tests/models/ctrl/test_modeling_tf_ctrl.py index fb48ba93eeee..c71c96bc9da9 100644 --- a/tests/models/ctrl/test_modeling_tf_ctrl.py +++ b/tests/models/ctrl/test_modeling_tf_ctrl.py @@ -185,6 +185,18 @@ class TFCTRLModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCase test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny + # config could not be created. + return True + + return False + def setUp(self): self.model_tester = TFCTRLModelTester(self) self.config_tester = ConfigTester(self, config_class=CTRLConfig, n_embd=37) diff --git a/tests/models/flaubert/test_modeling_flaubert.py b/tests/models/flaubert/test_modeling_flaubert.py index 0ac3ced935ac..40a9f68f69d1 100644 --- a/tests/models/flaubert/test_modeling_flaubert.py +++ b/tests/models/flaubert/test_modeling_flaubert.py @@ -390,6 +390,26 @@ class FlaubertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `FlaubertConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + # Flaubert has 2 QA models -> need to manually set the correct labels for one of them here def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/flaubert/test_modeling_tf_flaubert.py b/tests/models/flaubert/test_modeling_tf_flaubert.py index 996c7323ee55..117400203597 100644 --- a/tests/models/flaubert/test_modeling_tf_flaubert.py +++ b/tests/models/flaubert/test_modeling_tf_flaubert.py @@ -306,6 +306,26 @@ class TFFlaubertModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Test test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `FlaubertConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFFlaubertModelTester(self) self.config_tester = ConfigTester(self, config_class=FlaubertConfig, emb_dim=37) diff --git a/tests/models/fnet/test_modeling_fnet.py b/tests/models/fnet/test_modeling_fnet.py index 056d8237b1b5..836da260219f 100644 --- a/tests/models/fnet/test_modeling_fnet.py +++ b/tests/models/fnet/test_modeling_fnet.py @@ -299,6 +299,15 @@ class FNetModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_head_masking = False test_pruning = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "QAPipelineTests": + return True + + return False + # special case for ForPreTraining model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/gptj/test_modeling_gptj.py b/tests/models/gptj/test_modeling_gptj.py index 067b84ab7dee..5fe0fec39181 100644 --- a/tests/models/gptj/test_modeling_gptj.py +++ b/tests/models/gptj/test_modeling_gptj.py @@ -385,6 +385,22 @@ class GPTJModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_model_parallel = False test_head_masking = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + # special case for DoubleHeads model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/gptj/test_modeling_tf_gptj.py b/tests/models/gptj/test_modeling_tf_gptj.py index c23042973803..3aa63d2790a4 100644 --- a/tests/models/gptj/test_modeling_tf_gptj.py +++ b/tests/models/gptj/test_modeling_tf_gptj.py @@ -318,6 +318,22 @@ class TFGPTJModelTest(TFModelTesterMixin, TFCoreModelTesterMixin, PipelineTester test_missing_keys = False test_head_masking = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFGPTJModelTester(self) self.config_tester = ConfigTester(self, config_class=GPTJConfig, n_embd=37) diff --git a/tests/models/layoutlm/test_modeling_layoutlm.py b/tests/models/layoutlm/test_modeling_layoutlm.py index 0535fbf4e1f4..d2aad061c387 100644 --- a/tests/models/layoutlm/test_modeling_layoutlm.py +++ b/tests/models/layoutlm/test_modeling_layoutlm.py @@ -246,6 +246,20 @@ class LayoutLMModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase ) fx_compatible = True + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "DocumentQuestionAnsweringPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # This pipeline uses `sequence_ids()` which is only available for fast tokenizers. + return True + + return False + def setUp(self): self.model_tester = LayoutLMModelTester(self) self.config_tester = ConfigTester(self, config_class=LayoutLMConfig, hidden_size=37) diff --git a/tests/models/layoutlmv2/test_modeling_layoutlmv2.py b/tests/models/layoutlmv2/test_modeling_layoutlmv2.py index 812bff4aba00..6c82a34a626b 100644 --- a/tests/models/layoutlmv2/test_modeling_layoutlmv2.py +++ b/tests/models/layoutlmv2/test_modeling_layoutlmv2.py @@ -282,6 +282,29 @@ class LayoutLMv2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name in [ + "QAPipelineTests", + "TextClassificationPipelineTests", + "TokenClassificationPipelineTests", + "ZeroShotClassificationPipelineTests", + ]: + # `LayoutLMv2Config` was never used in pipeline tests (`test_pt_LayoutLMv2Config_XXX`) due to lack of tiny + # config. With new tiny model creation, it is available, but we need to fix the failed tests. + return True + elif ( + pipeline_test_casse_name == "DocumentQuestionAnsweringPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # This pipeline uses `sequence_ids()` which is only available for fast tokenizers. + return True + + return False + def setUp(self): self.model_tester = LayoutLMv2ModelTester(self) self.config_tester = ConfigTester(self, config_class=LayoutLMv2Config, hidden_size=37) diff --git a/tests/models/layoutlmv3/test_modeling_layoutlmv3.py b/tests/models/layoutlmv3/test_modeling_layoutlmv3.py index f62fff0048b0..5f1046d53481 100644 --- a/tests/models/layoutlmv3/test_modeling_layoutlmv3.py +++ b/tests/models/layoutlmv3/test_modeling_layoutlmv3.py @@ -298,6 +298,12 @@ class LayoutLMv3ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def setUp(self): self.model_tester = LayoutLMv3ModelTester(self) self.config_tester = ConfigTester(self, config_class=LayoutLMv3Config, hidden_size=37) diff --git a/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py b/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py index 09ca417595d6..df103194ab25 100644 --- a/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py +++ b/tests/models/layoutlmv3/test_modeling_tf_layoutlmv3.py @@ -291,6 +291,12 @@ class TFLayoutLMv3ModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Te test_resize_embeddings = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False) -> dict: inputs_dict = copy.deepcopy(inputs_dict) diff --git a/tests/models/lilt/test_modeling_lilt.py b/tests/models/lilt/test_modeling_lilt.py index e78d5d13d4e9..1bb92300c3db 100644 --- a/tests/models/lilt/test_modeling_lilt.py +++ b/tests/models/lilt/test_modeling_lilt.py @@ -244,6 +244,12 @@ class LiltModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin fx_compatible = False test_pruning = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def setUp(self): self.model_tester = LiltModelTester(self) self.config_tester = ConfigTester(self, config_class=LiltConfig, hidden_size=37) diff --git a/tests/models/longformer/test_modeling_longformer.py b/tests/models/longformer/test_modeling_longformer.py index 1dc97a114a9d..21853e44208f 100644 --- a/tests/models/longformer/test_modeling_longformer.py +++ b/tests/models/longformer/test_modeling_longformer.py @@ -326,6 +326,22 @@ class LongformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = LongformerModelTester(self) self.config_tester = ConfigTester(self, config_class=LongformerConfig, hidden_size=37) diff --git a/tests/models/longformer/test_modeling_tf_longformer.py b/tests/models/longformer/test_modeling_tf_longformer.py index ae11946cb9f6..b5452bc80ac5 100644 --- a/tests/models/longformer/test_modeling_tf_longformer.py +++ b/tests/models/longformer/test_modeling_tf_longformer.py @@ -299,6 +299,22 @@ class TFLongformerModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Te test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFLongformerModelTester(self) self.config_tester = ConfigTester(self, config_class=LongformerConfig, hidden_size=37) diff --git a/tests/models/m2m_100/test_modeling_m2m_100.py b/tests/models/m2m_100/test_modeling_m2m_100.py index 2457eea21ad2..e37a0233076a 100644 --- a/tests/models/m2m_100/test_modeling_m2m_100.py +++ b/tests/models/m2m_100/test_modeling_m2m_100.py @@ -246,6 +246,17 @@ class M2M100ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TranslationPipelineTests": + # Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`. + # `M2M100Config` was never used in pipeline tests: cannot create a simple tokenizer. + return True + + return False + def setUp(self): self.model_tester = M2M100ModelTester(self) self.config_tester = ConfigTester(self, config_class=M2M100Config) diff --git a/tests/models/mvp/test_modeling_mvp.py b/tests/models/mvp/test_modeling_mvp.py index 5513f34b4607..7ade83165902 100644 --- a/tests/models/mvp/test_modeling_mvp.py +++ b/tests/models/mvp/test_modeling_mvp.py @@ -433,6 +433,22 @@ class MvpModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = MvpModelTester(self) self.config_tester = ConfigTester(self, config_class=MvpConfig) diff --git a/tests/models/openai/test_modeling_openai.py b/tests/models/openai/test_modeling_openai.py index a9d76d3770b7..0e8ba6d9ced7 100644 --- a/tests/models/openai/test_modeling_openai.py +++ b/tests/models/openai/test_modeling_openai.py @@ -210,6 +210,18 @@ class OpenAIGPTModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a + # tiny config could not be created. + return True + + return False + # special case for DoubleHeads model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/openai/test_modeling_tf_openai.py b/tests/models/openai/test_modeling_tf_openai.py index f059487f21a3..a4cf71bf1a9f 100644 --- a/tests/models/openai/test_modeling_tf_openai.py +++ b/tests/models/openai/test_modeling_tf_openai.py @@ -214,6 +214,18 @@ class TFOpenAIGPTModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Tes test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests": + # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. + # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a + # tiny config could not be created. + return True + + return False + def setUp(self): self.model_tester = TFOpenAIGPTModelTester(self) self.config_tester = ConfigTester(self, config_class=OpenAIGPTConfig, n_embd=37) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 4805f1c215a7..f8f217790c2c 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -207,6 +207,22 @@ class OPTModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = OPTModelTester(self) self.config_tester = ConfigTester(self, config_class=OPTConfig) diff --git a/tests/models/plbart/test_modeling_plbart.py b/tests/models/plbart/test_modeling_plbart.py index 28278f9a4669..4c2b2b84ad82 100644 --- a/tests/models/plbart/test_modeling_plbart.py +++ b/tests/models/plbart/test_modeling_plbart.py @@ -237,6 +237,17 @@ class PLBartModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMix test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TranslationPipelineTests": + # Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`. + # `PLBartConfig` was never used in pipeline tests: cannot create a simple tokenizer. + return True + + return False + def setUp(self): self.model_tester = PLBartModelTester(self) self.config_tester = ConfigTester(self, config_class=PLBartConfig) diff --git a/tests/models/prophetnet/test_modeling_prophetnet.py b/tests/models/prophetnet/test_modeling_prophetnet.py index 2579be5396ed..1d4b45e23c90 100644 --- a/tests/models/prophetnet/test_modeling_prophetnet.py +++ b/tests/models/prophetnet/test_modeling_prophetnet.py @@ -904,6 +904,18 @@ class ProphetNetModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTeste test_resize_embeddings = False is_encoder_decoder = True + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `ProphetNetConfig` was never used in pipeline tests: cannot create a simple + # tokenizer. + return True + + return False + def setUp(self): self.model_tester = ProphetNetModelTester(self) self.config_tester = ConfigTester(self, config_class=ProphetNetConfig) diff --git a/tests/models/reformer/test_modeling_reformer.py b/tests/models/reformer/test_modeling_reformer.py index 4793f4ea4315..a7f4f2f45416 100644 --- a/tests/models/reformer/test_modeling_reformer.py +++ b/tests/models/reformer/test_modeling_reformer.py @@ -709,6 +709,22 @@ class ReformerLSHAttnModelTest( test_headmasking = False test_torchscript = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = ReformerModelTester( self, diff --git a/tests/models/roc_bert/test_modeling_roc_bert.py b/tests/models/roc_bert/test_modeling_roc_bert.py index afc0f63122d2..2efd9b79926f 100644 --- a/tests/models/roc_bert/test_modeling_roc_bert.py +++ b/tests/models/roc_bert/test_modeling_roc_bert.py @@ -586,6 +586,24 @@ class RoCBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) else {} ) + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name in [ + "FillMaskPipelineTests", + "FeatureExtractionPipelineTests", + "TextClassificationPipelineTests", + "TokenClassificationPipelineTests", + ]: + # Get error: IndexError: index out of range in self. + # `word_shape_file` and `word_pronunciation_file` should be shrunk during tiny model creation, + # otherwise `IndexError` could occur in some embedding layers. Skip for now until this model has + # more usage. + return True + + return False + # special case for ForPreTraining model def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/roformer/test_modeling_tf_roformer.py b/tests/models/roformer/test_modeling_tf_roformer.py index f5f3875db1c5..0a632e39a2ed 100644 --- a/tests/models/roformer/test_modeling_tf_roformer.py +++ b/tests/models/roformer/test_modeling_tf_roformer.py @@ -271,6 +271,15 @@ class TFRoFormerModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Test test_head_masking = False test_onnx = False + # TODO: add `prepare_inputs_for_generation` for `TFRoFormerForCausalLM` + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + return True + + return False + def setUp(self): self.model_tester = TFRoFormerModelTester(self) self.config_tester = ConfigTester(self, config_class=RoFormerConfig, hidden_size=37) diff --git a/tests/models/tapas/test_modeling_tapas.py b/tests/models/tapas/test_modeling_tapas.py index b3b9217a0019..644307e3f917 100644 --- a/tests/models/tapas/test_modeling_tapas.py +++ b/tests/models/tapas/test_modeling_tapas.py @@ -486,6 +486,12 @@ def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): ) return inputs_dict + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def setUp(self): self.model_tester = TapasModelTester(self) self.config_tester = ConfigTester(self, config_class=TapasConfig, dim=37) diff --git a/tests/models/tapas/test_modeling_tf_tapas.py b/tests/models/tapas/test_modeling_tf_tapas.py index 380b7605b894..c3cc5fae3a9c 100644 --- a/tests/models/tapas/test_modeling_tf_tapas.py +++ b/tests/models/tapas/test_modeling_tf_tapas.py @@ -443,6 +443,12 @@ class TFTapasModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCas test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False) -> dict: inputs_dict = copy.deepcopy(inputs_dict) diff --git a/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py b/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py index 88bfbc1c7bb9..47880013b97e 100644 --- a/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py +++ b/tests/models/transfo_xl/test_modeling_tf_transfo_xl.py @@ -177,6 +177,18 @@ class TFTransfoXLModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.Tes test_onnx = False test_mismatched_shapes = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple + # tokenizer. + return True + + return False + def setUp(self): self.model_tester = TFTransfoXLModelTester(self) self.config_tester = ConfigTester(self, config_class=TransfoXLConfig, d_embed=37) diff --git a/tests/models/transfo_xl/test_modeling_transfo_xl.py b/tests/models/transfo_xl/test_modeling_transfo_xl.py index df9647ae209e..c86cd704f159 100644 --- a/tests/models/transfo_xl/test_modeling_transfo_xl.py +++ b/tests/models/transfo_xl/test_modeling_transfo_xl.py @@ -271,6 +271,18 @@ class TransfoXLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTester test_resize_embeddings = True test_mismatched_shapes = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple + # tokenizer. + return True + + return False + def check_cutoffs_and_n_token( self, copied_cutoffs, layer, model_embed, model, model_class, resized_value, vocab_size ): diff --git a/tests/models/xlm/test_modeling_tf_xlm.py b/tests/models/xlm/test_modeling_tf_xlm.py index e34978495c5a..3f27dad5d4f7 100644 --- a/tests/models/xlm/test_modeling_tf_xlm.py +++ b/tests/models/xlm/test_modeling_tf_xlm.py @@ -309,6 +309,26 @@ class TFXLMModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `XLMConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + def setUp(self): self.model_tester = TFXLMModelTester(self) self.config_tester = ConfigTester(self, config_class=XLMConfig, emb_dim=37) diff --git a/tests/models/xlm/test_modeling_xlm.py b/tests/models/xlm/test_modeling_xlm.py index 58d2a2646613..c86210e599ea 100644 --- a/tests/models/xlm/test_modeling_xlm.py +++ b/tests/models/xlm/test_modeling_xlm.py @@ -391,6 +391,26 @@ class XLMModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FillMaskPipelineTests": + # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. + # `XLMConfig` was never used in pipeline tests: cannot create a simple tokenizer + return True + elif ( + pipeline_test_casse_name == "QAPipelineTests" + and tokenizer_name is not None + and not tokenizer_name.endswith("Fast") + ): + # `QAPipelineTests` fails for a few models when the slower tokenizer are used. + # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) + # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer + return True + + return False + # XLM has 2 QA models -> need to manually set the correct labels for one of them here def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/test_pipeline_mixin.py b/tests/test_pipeline_mixin.py index cef7062ba6d5..fbeac086e110 100644 --- a/tests/test_pipeline_mixin.py +++ b/tests/test_pipeline_mixin.py @@ -172,7 +172,7 @@ def run_model_pipeline_tests(self, task, repo_name, model_architecture, tokenize for tokenizer_name in tokenizer_names: for processor_name in processor_names: - if is_test_to_skip( + if self.is_pipeline_test_to_skip( pipeline_test_class_name, model_architecture.config_class, model_architecture, @@ -404,6 +404,11 @@ def test_pipeline_zero_shot_image_classification(self): def test_pipeline_zero_shot_object_detection(self): self.run_task_tests(task="zero-shot-object-detection") + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return False + def validate_test_components(test_case, task, model, tokenizer, processor): # TODO: Move this to tiny model creation script @@ -431,109 +436,3 @@ def validate_test_components(test_case, task, model, tokenizer, processor): f"(`{tokenizer.__class__.__name__}`) has {len(tokenizer)} tokens which is greater than " f"`config_vocab_size` ({config_vocab_size}). Something is wrong." ) - - -def is_test_to_skip(test_casse_name, config_class, model_architecture, tokenizer_name, processor_name): - """Some tests are just not working""" - - to_skip = False - - if config_class.__name__ == "RoCBertConfig" and test_casse_name in [ - "FillMaskPipelineTests", - "FeatureExtractionPipelineTests", - "TextClassificationPipelineTests", - "TokenClassificationPipelineTests", - ]: - # Get error: IndexError: index out of range in self. - # `word_shape_file` and `word_pronunciation_file` should be shrunk during tiny model creation, - # otherwise `IndexError` could occur in some embedding layers. Skip for now until this model has - # more usage. - to_skip = True - elif config_class.__name__ in ["LayoutLMv3Config", "LiltConfig"]: - # Get error: ValueError: Words must be of type `List[str]`. Previously, `LayoutLMv3` is not - # used in pipeline tests as it could not find a checkpoint - # TODO: check and fix if possible - to_skip = True - # config/model class we decide to skip - elif config_class.__name__ in ["TapasConfig"]: - # Get error: AssertionError: Table must be of type pd.DataFrame. Also, the tiny model has large - # vocab size as the fast tokenizer could not be converted. Previous, `Tapas` is not used in - # pipeline tests due to the same reason. - # TODO: check and fix if possible - to_skip = True - - # TODO: check and fix if possible - if not to_skip and tokenizer_name is not None: - if ( - test_casse_name == "QAPipelineTests" - and not tokenizer_name.endswith("Fast") - and config_class.__name__ - in [ - "FlaubertConfig", - "GPTJConfig", - "LongformerConfig", - "MvpConfig", - "OPTConfig", - "ReformerConfig", - "XLMConfig", - ] - ): - # `QAPipelineTests` fails for a few models when the slower tokenizer are used. - # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework) - # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer - to_skip = True - elif test_casse_name == "ZeroShotClassificationPipelineTests" and config_class.__name__ in [ - "CTRLConfig", - "OpenAIGPTConfig", - ]: - # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers. - # `CTRLConfig` and `OpenAIGPTConfig` were never used in pipeline tests, either because of a missing - # checkpoint or because a tiny config could not be created - to_skip = True - elif test_casse_name == "TranslationPipelineTests" and config_class.__name__ in [ - "M2M100Config", - "PLBartConfig", - ]: - # Get `ValueError: Translation requires a `src_lang` and a `tgt_lang` for this model`. - # `M2M100Config` and `PLBartConfig` were never used in pipeline tests: cannot create a simple tokenizer - to_skip = True - elif test_casse_name == "TextGenerationPipelineTests" and config_class.__name__ in [ - "ProphetNetConfig", - "TransfoXLConfig", - ]: - # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. - # `TransfoXLConfig` and `ProphetNetConfig` were never used in pipeline tests: cannot create a simple - # tokenizer. - to_skip = True - elif test_casse_name == "FillMaskPipelineTests" and config_class.__name__ in [ - "FlaubertConfig", - "XLMConfig", - ]: - # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`. - # `FlaubertConfig` and `TransfoXLConfig` were never used in pipeline tests: cannot create a simple - # tokenizer - to_skip = True - elif test_casse_name == "TextGenerationPipelineTests" and model_architecture.__name__ in [ - "TFRoFormerForCausalLM" - ]: - # TODO: add `prepare_inputs_for_generation` for `TFRoFormerForCausalLM` - to_skip = True - elif test_casse_name == "QAPipelineTests" and model_architecture.__name__ in ["FNetForQuestionAnswering"]: - # TODO: The change in `base.py` in the PR #21132 (https://github.com/huggingface/transformers/pull/21132) - # fails this test case. Skip for now - a fix for this along with the initial changes in PR #20426 is - # too much. Let `ydshieh` to fix it ASAP once #20426 is merged. - to_skip = True - elif config_class.__name__ == "LayoutLMv2Config" and test_casse_name in [ - "QAPipelineTests", - "TextClassificationPipelineTests", - "TokenClassificationPipelineTests", - "ZeroShotClassificationPipelineTests", - ]: - # `LayoutLMv2Config` was never used in pipeline tests (`test_pt_LayoutLMv2Config_XXX`) due to lack of tiny - # config. With new tiny model creation, it is available, but we need to fix the failed tests. - to_skip = True - elif test_casse_name == "DocumentQuestionAnsweringPipelineTests" and not tokenizer_name.endswith("Fast"): - # This pipeline uses `sequence_ids()` which is only available for fast tokenizers. - to_skip = True - - return to_skip From 86b33f303cfd4b708518aec4efa4f1e8f6e794b3 Mon Sep 17 00:00:00 2001 From: Alara Dirik <8944735+alaradirik@users.noreply.github.com> Date: Tue, 14 Mar 2023 12:08:14 +0300 Subject: [PATCH 091/277] Add ConvNeXT V2 (#21679) * Add ConvNeXt V2 to transformers * TF model is separated from the PR to fix issues --- README.md | 1 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/de/index.mdx | 1 + docs/source/en/_toctree.yml | 2 + docs/source/en/index.mdx | 2 + docs/source/en/model_doc/convnextv2.mdx | 57 ++ docs/source/en/tasks/image_classification.mdx | 2 +- docs/source/es/index.mdx | 1 + docs/source/es/serialization.mdx | 1 + docs/source/fr/index.mdx | 1 + docs/source/it/index.mdx | 1 + docs/source/ja/index.mdx | 1 + docs/source/ko/index.mdx | 1 + docs/source/pt/index.mdx | 1 + docs/source/pt/serialization.mdx | 1 + docs/source/zh/index.mdx | 1 + src/transformers/__init__.py | 18 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + .../models/auto/image_processing_auto.py | 1 + src/transformers/models/auto/modeling_auto.py | 3 + .../convnext/image_processing_convnext.py | 2 +- .../models/convnextv2/__init__.py | 73 +++ .../convnextv2/configuration_convnextv2.py | 114 ++++ .../convert_convnextv2_to_pytorch.py | 282 +++++++++ .../models/convnextv2/modeling_convnextv2.py | 593 ++++++++++++++++++ src/transformers/utils/dummy_pt_objects.py | 31 + tests/models/convnextv2/__init__.py | 0 .../convnextv2/test_modeling_convnextv2.py | 348 ++++++++++ utils/check_repo.py | 1 + 35 files changed, 1548 insertions(+), 2 deletions(-) create mode 100644 docs/source/en/model_doc/convnextv2.mdx create mode 100644 src/transformers/models/convnextv2/__init__.py create mode 100644 src/transformers/models/convnextv2/configuration_convnextv2.py create mode 100644 src/transformers/models/convnextv2/convert_convnextv2_to_pytorch.py create mode 100644 src/transformers/models/convnextv2/modeling_convnextv2.py create mode 100644 tests/models/convnextv2/__init__.py create mode 100644 tests/models/convnextv2/test_modeling_convnextv2.py diff --git a/README.md b/README.md index cd866a4ce42c..17df3295ca9a 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. diff --git a/README_es.md b/README_es.md index 3027a03caca7..2ae2a604a5de 100644 --- a/README_es.md +++ b/README_es.md @@ -296,6 +296,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. diff --git a/README_hd.md b/README_hd.md index fc5fdd882c66..9dffd0be1b77 100644 --- a/README_hd.md +++ b/README_hd.md @@ -268,6 +268,7 @@ conda install -c huggingface transformers 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (माइक्रोसॉफ्ट रिसर्च एशिया से) कागज के साथ [फास्ट ट्रेनिंग कन्वर्जेंस के लिए सशर्त डीईटीआर](https://arxiv. org/abs/2108.06152) डेपू मेंग, ज़ियाओकांग चेन, ज़ेजिया फैन, गैंग ज़ेंग, होउकियांग ली, युहुई युआन, लेई सन, जिंगडोंग वांग द्वारा। 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (YituTech से) साथ में कागज [ConvBERT: स्पैन-आधारित डायनेमिक कनवल्शन के साथ BERT में सुधार](https://arxiv .org/abs/2008.02496) जिहांग जियांग, वीहाओ यू, डाकान झोउ, युनपेंग चेन, जियाशी फेंग, शुइचेंग यान द्वारा। 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (Facebook AI से) साथ वाला पेपर [A ConvNet for the 2020s](https://arxiv.org/abs /2201.03545) ज़ुआंग लियू, हेंज़ी माओ, चाओ-युआन वू, क्रिस्टोफ़ फीचटेनहोफ़र, ट्रेवर डेरेल, सैनिंग ज़ी द्वारा। +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (सिंघुआ यूनिवर्सिटी से) साथ में पेपर [सीपीएम: ए लार्ज-स्केल जेनेरेटिव चाइनीज प्री-ट्रेंड लैंग्वेज मॉडल](https : //arxiv.org/abs/2012.00413) झेंग्यान झांग, जू हान, हाओ झोउ, पेई के, युक्सियन गु, डेमिंग ये, युजिया किन, युशेंग सु, हाओझे जी, जियान गुआन, फैंचाओ क्यूई, ज़ियाओझी वांग, यानान झेंग द्वारा , गुओयांग ज़ेंग, हुआनकी काओ, शेंगकी चेन, डाइक्सुआन ली, ज़ेनबो सन, ज़ियुआन लियू, मिनली हुआंग, वेंटाओ हान, जी तांग, जुआनज़ी ली, ज़ियाओयान झू, माओसोंग सन। 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (सेल्सफोर्स से) साथ में पेपर [CTRL: ए कंडिशनल ट्रांसफॉर्मर लैंग्वेज मॉडल फॉर कंट्रोलेबल जेनरेशन](https://arxiv.org/abs/1909.05858) नीतीश शिरीष केसकर*, ब्रायन मैककैन*, लव आर. वार्ष्णेय, कैमिंग जिओंग और रिचर्ड द्वारा सोचर द्वारा जारी किया गया। 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (Microsoft से) साथ में दिया गया पेपर [CvT: इंट्रोड्यूसिंग कनवॉल्यूशन टू विजन ट्रांसफॉर्मर्स](https://arxiv.org/ एब्स/2103.15808) हैपिंग वू, बिन जिओ, नोएल कोडेला, मेंगचेन लियू, जियांग दाई, लू युआन, लेई झांग द्वारा। diff --git a/README_ja.md b/README_ja.md index 3fae87a294fc..d08684c0366d 100644 --- a/README_ja.md +++ b/README_ja.md @@ -330,6 +330,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (Microsoft Research Asia から) Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang から公開された研究論文: [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (YituTech から) Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan から公開された研究論文: [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (Facebook AI から) Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie から公開された研究論文: [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (Tsinghua University から) Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun から公開された研究論文: [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (Salesforce から) Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher から公開された研究論文: [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (Microsoft から) Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang から公開された研究論文: [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) diff --git a/README_ko.md b/README_ko.md index c19fabca5c90..d68a9b54e2d3 100644 --- a/README_ko.md +++ b/README_ko.md @@ -245,6 +245,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (Microsoft Research Asia 에서) Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang 의 [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) 논문과 함께 발표했습니다. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (YituTech 에서) Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan 의 [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) 논문과 함께 발표했습니다. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (Facebook AI 에서) Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie 의 [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) 논문과 함께 발표했습니다. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (Tsinghua University 에서) Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun 의 [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) 논문과 함께 발표했습니다. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (Salesforce 에서) Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher 의 [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) 논문과 함께 발표했습니다. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (Microsoft 에서) Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang 의 [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) 논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index f077fbca40b8..2e8f9464e06c 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -269,6 +269,7 @@ conda install -c huggingface transformers 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (来自 Microsoft Research Asia) 伴随论文 [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) 由 Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang 发布。 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (来自 YituTech) 伴随论文 [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) 由 Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan 发布。 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (来自 Facebook AI) 伴随论文 [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) 由 Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie 发布。 +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (来自 Tsinghua University) 伴随论文 [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) 由 Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun 发布。 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (来自 Salesforce) 伴随论文 [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) 由 Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher 发布。 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (来自 Microsoft) 伴随论文 [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) 由 Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index cc19893bc9ee..7fa3f4a6c3a0 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -281,6 +281,7 @@ conda install -c huggingface transformers 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. diff --git a/docs/source/de/index.mdx b/docs/source/de/index.mdx index 0e2b3fb68780..c1340820b567 100644 --- a/docs/source/de/index.mdx +++ b/docs/source/de/index.mdx @@ -73,6 +73,7 @@ Die Bibliothek enthält derzeit JAX-, PyTorch- und TensorFlow-Implementierungen, 1. **[CodeGen](model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://arxiv.org/abs/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong. 1. **[ConvBERT](model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index eddfc48d5cd5..947a40978294 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -430,6 +430,8 @@ title: Conditional DETR - local: model_doc/convnext title: ConvNeXT + - local: model_doc/convnextv2 + title: ConvNeXTV2 - local: model_doc/cvt title: CvT - local: model_doc/deformable_detr diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 36964e924a81..1f4f5a0d43f6 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -82,6 +82,7 @@ The documentation is organized into five sections: 1. **[Conditional DETR](model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. @@ -278,6 +279,7 @@ Flax), PyTorch, and/or TensorFlow. | Conditional DETR | ❌ | ❌ | ✅ | ❌ | ❌ | | ConvBERT | ✅ | ✅ | ✅ | ✅ | ❌ | | ConvNeXT | ❌ | ❌ | ✅ | ✅ | ❌ | +| ConvNeXTV2 | ❌ | ❌ | ✅ | ❌ | ❌ | | CTRL | ✅ | ❌ | ✅ | ✅ | ❌ | | CvT | ❌ | ❌ | ✅ | ✅ | ❌ | | Data2VecAudio | ❌ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/model_doc/convnextv2.mdx b/docs/source/en/model_doc/convnextv2.mdx new file mode 100644 index 000000000000..6572854bfa6c --- /dev/null +++ b/docs/source/en/model_doc/convnextv2.mdx @@ -0,0 +1,57 @@ + + +# ConvNeXt V2 + +## Overview + +The ConvNeXt V2 model was proposed in [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +ConvNeXt V2 is a pure convolutional model (ConvNet), inspired by the design of Vision Transformers, and a successor of [ConvNeXT](convnext). + +The abstract from the paper is the following: + +*Driven by improved architectures and better representation learning frameworks, the field of visual recognition has enjoyed rapid modernization and performance boost in the early 2020s. For example, modern ConvNets, represented by ConvNeXt, have demonstrated strong performance in various scenarios. While these models were originally designed for supervised learning with ImageNet labels, they can also potentially benefit from self-supervised learning techniques such as masked autoencoders (MAE). However, we found that simply combining these two approaches leads to subpar performance. In this paper, we propose a fully convolutional masked autoencoder framework and a new Global Response Normalization (GRN) layer that can be added to the ConvNeXt architecture to enhance inter-channel feature competition. This co-design of self-supervised learning techniques and architectural improvement results in a new model family called ConvNeXt V2, which significantly improves the performance of pure ConvNets on various recognition benchmarks, including ImageNet classification, COCO detection, and ADE20K segmentation. We also provide pre-trained ConvNeXt V2 models of various sizes, ranging from an efficient 3.7M-parameter Atto model with 76.7% top-1 accuracy on ImageNet, to a 650M Huge model that achieves a state-of-the-art 88.9% accuracy using only public training data.* + +Tips: + +- See the code examples below each model regarding usage. + + + + ConvNeXt V2 architecture. Taken from the original paper. + +This model was contributed by [adirik](https://huggingface.co/adirik). The original code can be found [here](https://github.com/facebookresearch/ConvNeXt-V2). + +## Resources + +A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with ConvNeXt V2. + + + +- [`ConvNextV2ForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). + +If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. + +## ConvNextV2Config + +[[autodoc]] ConvNextV2Config + +## ConvNextV2Model + +[[autodoc]] ConvNextV2Model + - forward + +## ConvNextV2ForImageClassification + +[[autodoc]] ConvNextV2ForImageClassification + - forward \ No newline at end of file diff --git a/docs/source/en/tasks/image_classification.mdx b/docs/source/en/tasks/image_classification.mdx index 4559b7d1af21..c3faa6145e16 100644 --- a/docs/source/en/tasks/image_classification.mdx +++ b/docs/source/en/tasks/image_classification.mdx @@ -30,7 +30,7 @@ The task illustrated in this tutorial is supported by the following model archit -[BEiT](../model_doc/beit), [BiT](../model_doc/bit), [ConvNeXT](../model_doc/convnext), [CvT](../model_doc/cvt), [Data2VecVision](../model_doc/data2vec-vision), [DeiT](../model_doc/deit), [DiNAT](../model_doc/dinat), [EfficientFormer](../model_doc/efficientformer), [EfficientNet](../model_doc/efficientnet), [ImageGPT](../model_doc/imagegpt), [LeViT](../model_doc/levit), [MobileNetV1](../model_doc/mobilenet_v1), [MobileNetV2](../model_doc/mobilenet_v2), [MobileViT](../model_doc/mobilevit), [NAT](../model_doc/nat), [Perceiver](../model_doc/perceiver), [PoolFormer](../model_doc/poolformer), [RegNet](../model_doc/regnet), [ResNet](../model_doc/resnet), [SegFormer](../model_doc/segformer), [Swin Transformer](../model_doc/swin), [Swin Transformer V2](../model_doc/swinv2), [VAN](../model_doc/van), [ViT](../model_doc/vit), [ViT Hybrid](../model_doc/vit_hybrid), [ViTMSN](../model_doc/vit_msn) +[BEiT](../model_doc/beit), [BiT](../model_doc/bit), [ConvNeXT](../model_doc/convnext), [ConvNeXTV2](../model_doc/convnextv2), [CvT](../model_doc/cvt), [Data2VecVision](../model_doc/data2vec-vision), [DeiT](../model_doc/deit), [DiNAT](../model_doc/dinat), [EfficientFormer](../model_doc/efficientformer), [EfficientNet](../model_doc/efficientnet), [ImageGPT](../model_doc/imagegpt), [LeViT](../model_doc/levit), [MobileNetV1](../model_doc/mobilenet_v1), [MobileNetV2](../model_doc/mobilenet_v2), [MobileViT](../model_doc/mobilevit), [NAT](../model_doc/nat), [Perceiver](../model_doc/perceiver), [PoolFormer](../model_doc/poolformer), [RegNet](../model_doc/regnet), [ResNet](../model_doc/resnet), [SegFormer](../model_doc/segformer), [Swin Transformer](../model_doc/swin), [Swin Transformer V2](../model_doc/swinv2), [VAN](../model_doc/van), [ViT](../model_doc/vit), [ViT Hybrid](../model_doc/vit_hybrid), [ViTMSN](../model_doc/vit_msn) diff --git a/docs/source/es/index.mdx b/docs/source/es/index.mdx index d9a7a26d2463..baedf45a0fdc 100644 --- a/docs/source/es/index.mdx +++ b/docs/source/es/index.mdx @@ -63,6 +63,7 @@ La biblioteca actualmente contiene implementaciones de JAX, PyTorch y TensorFlow 1. **[CamemBERT](model_doc/camembert)** (de Inria/Facebook/Sorbonne) publicado con el paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) por Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah y Benoît Sagot. 1. **[CANINE](model_doc/canine)** (de Google Research) publicado con el paper [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) por Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting. 1. **[ConvNeXT](model_doc/convnext)** (de Facebook AI) publicado con el paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) por Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (de Facebook AI) publicado con el paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) por Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CLIP](model_doc/clip)** (de OpenAI) publicado con el paper [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) por Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. 1. **[ConvBERT](model_doc/convbert)** (de YituTech) publicado con el paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) por Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[CPM](model_doc/cpm)** (de Universidad de Tsinghua) publicado con el paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) por Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. diff --git a/docs/source/es/serialization.mdx b/docs/source/es/serialization.mdx index 4c42fd5d830e..2815734bfa79 100644 --- a/docs/source/es/serialization.mdx +++ b/docs/source/es/serialization.mdx @@ -59,6 +59,7 @@ Las configuraciones a la medida incluyen las siguientes arquitecturas: - CodeGen - ConvBERT - ConvNeXT +- ConvNeXTV2 - Data2VecText - Data2VecVision - DeBERTa diff --git a/docs/source/fr/index.mdx b/docs/source/fr/index.mdx index a2caf6309f43..8184d1ced8b0 100644 --- a/docs/source/fr/index.mdx +++ b/docs/source/fr/index.mdx @@ -80,6 +80,7 @@ La documentation est organisée en 5 parties: 1. **[Conditional DETR](model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. diff --git a/docs/source/it/index.mdx b/docs/source/it/index.mdx index e99c64fd1d83..a6d474bb6c23 100644 --- a/docs/source/it/index.mdx +++ b/docs/source/it/index.mdx @@ -68,6 +68,7 @@ La libreria attualmente contiene implementazioni in JAX, PyTorch e TensorFlow, p 1. **[CamemBERT](model_doc/camembert)** (da Inria/Facebook/Sorbonne) rilasciato con il paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) da Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah e Benoît Sagot. 1. **[CANINE](model_doc/canine)** (da Google Research) rilasciato con il paper [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) da Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting. 1. **[ConvNeXT](model_doc/convnext)** (da Facebook AI) rilasciato con il paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) da Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (da Facebook AI) rilasciato con il paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) da Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CLIP](model_doc/clip)** (da OpenAI) rilasciato con il paper [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) da Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. 1. **[ConvBERT](model_doc/convbert)** (da YituTech) rilasciato con il paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) da Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[CPM](model_doc/cpm)** (dalla Università di Tsinghua) rilasciato con il paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) da Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. diff --git a/docs/source/ja/index.mdx b/docs/source/ja/index.mdx index d98db263461b..a644a885d32d 100644 --- a/docs/source/ja/index.mdx +++ b/docs/source/ja/index.mdx @@ -78,6 +78,7 @@ specific language governing permissions and limitations under the License. 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (Microsoft Research Asia から) Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang から公開された研究論文: [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (YituTech から) Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan から公開された研究論文: [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (Facebook AI から) Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie から公開された研究論文: [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) +1. **[ConvNeXTV2](model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (Tsinghua University から) Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun から公開された研究論文: [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (Salesforce から) Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher から公開された研究論文: [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (Microsoft から) Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang から公開された研究論文: [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) diff --git a/docs/source/ko/index.mdx b/docs/source/ko/index.mdx index 8073c2daeb83..789aa41a28e0 100644 --- a/docs/source/ko/index.mdx +++ b/docs/source/ko/index.mdx @@ -72,6 +72,7 @@ specific language governing permissions and limitations under the License. 1. **[Conditional DETR](model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. diff --git a/docs/source/pt/index.mdx b/docs/source/pt/index.mdx index bdbd7385fdcf..8e74a1feb4fe 100644 --- a/docs/source/pt/index.mdx +++ b/docs/source/pt/index.mdx @@ -76,6 +76,7 @@ Atualmente a biblioteca contém implementações do PyTorch, TensorFlow e JAX, p 1. **[CamemBERT](model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 1. **[CANINE](model_doc/canine)** (from Google Research) released with the paper [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) by Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting. 1. **[ConvNeXT](model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CLIP](model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. 1. **[ConvBERT](model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[CPM](model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. diff --git a/docs/source/pt/serialization.mdx b/docs/source/pt/serialization.mdx index 2a01640be467..5e95d5951215 100644 --- a/docs/source/pt/serialization.mdx +++ b/docs/source/pt/serialization.mdx @@ -60,6 +60,7 @@ As configurações prontas incluem as seguintes arquiteturas: - Conditional DETR - ConvBERT - ConvNeXT +- ConvNeXTV2 - Data2VecText - Data2VecVision - DeBERTa diff --git a/docs/source/zh/index.mdx b/docs/source/zh/index.mdx index 4d69d590c692..33b9181b71d4 100644 --- a/docs/source/zh/index.mdx +++ b/docs/source/zh/index.mdx @@ -78,6 +78,7 @@ specific language governing permissions and limitations under the License. 1. **[Conditional DETR](model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. +1. **[ConvNeXTV2](model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 8d41feb8abd2..45fe5447b31b 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -241,6 +241,7 @@ "models.conditional_detr": ["CONDITIONAL_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP", "ConditionalDetrConfig"], "models.convbert": ["CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "ConvBertConfig", "ConvBertTokenizer"], "models.convnext": ["CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP", "ConvNextConfig"], + "models.convnextv2": ["CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP", "ConvNextV2Config"], "models.cpm": [], "models.ctrl": ["CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP", "CTRLConfig", "CTRLTokenizer"], "models.cvt": ["CVT_PRETRAINED_CONFIG_ARCHIVE_MAP", "CvtConfig"], @@ -1300,6 +1301,15 @@ "ConvNextPreTrainedModel", ] ) + _import_structure["models.convnextv2"].extend( + [ + "CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST", + "ConvNextV2Backbone", + "ConvNextV2ForImageClassification", + "ConvNextV2Model", + "ConvNextV2PreTrainedModel", + ] + ) _import_structure["models.ctrl"].extend( [ "CTRL_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -3852,6 +3862,7 @@ from .models.conditional_detr import CONDITIONAL_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP, ConditionalDetrConfig from .models.convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertTokenizer from .models.convnext import CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvNextConfig + from .models.convnextv2 import CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvNextV2Config from .models.ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig, CTRLTokenizer from .models.cvt import CVT_PRETRAINED_CONFIG_ARCHIVE_MAP, CvtConfig from .models.data2vec import ( @@ -4778,6 +4789,13 @@ ConvNextModel, ConvNextPreTrainedModel, ) + from .models.convnextv2 import ( + CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST, + ConvNextV2Backbone, + ConvNextV2ForImageClassification, + ConvNextV2Model, + ConvNextV2PreTrainedModel, + ) from .models.ctrl import ( CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, CTRLForSequenceClassification, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 29d023fc6a5d..9951da553807 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -48,6 +48,7 @@ conditional_detr, convbert, convnext, + convnextv2, cpm, ctrl, cvt, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 1ab0b62c793a..70ab1bc9b10c 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -57,6 +57,7 @@ ("conditional_detr", "ConditionalDetrConfig"), ("convbert", "ConvBertConfig"), ("convnext", "ConvNextConfig"), + ("convnextv2", "ConvNextV2Config"), ("ctrl", "CTRLConfig"), ("cvt", "CvtConfig"), ("data2vec-audio", "Data2VecAudioConfig"), @@ -236,6 +237,7 @@ ("conditional_detr", "CONDITIONAL_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("convbert", "CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("convnext", "CONVNEXT_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("convnextv2", "CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("ctrl", "CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("cvt", "CVT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("data2vec-audio", "DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -405,6 +407,7 @@ ("conditional_detr", "Conditional DETR"), ("convbert", "ConvBERT"), ("convnext", "ConvNeXT"), + ("convnextv2", "ConvNeXTV2"), ("cpm", "CPM"), ("ctrl", "CTRL"), ("cvt", "CvT"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 6274c88f7d31..7a042c56c5c7 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -48,6 +48,7 @@ ("clipseg", "ViTImageProcessor"), ("conditional_detr", "ConditionalDetrImageProcessor"), ("convnext", "ConvNextImageProcessor"), + ("convnextv2", "ConvNextImageProcessor"), ("cvt", "ConvNextImageProcessor"), ("data2vec-vision", "BeitImageProcessor"), ("deformable_detr", "DeformableDetrImageProcessor"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index f871de8e55f2..c29965944aaa 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -56,6 +56,7 @@ ("conditional_detr", "ConditionalDetrModel"), ("convbert", "ConvBertModel"), ("convnext", "ConvNextModel"), + ("convnextv2", "ConvNextV2Model"), ("ctrl", "CTRLModel"), ("cvt", "CvtModel"), ("data2vec-audio", "Data2VecAudioModel"), @@ -410,6 +411,7 @@ ("beit", "BeitForImageClassification"), ("bit", "BitForImageClassification"), ("convnext", "ConvNextForImageClassification"), + ("convnextv2", "ConvNextV2ForImageClassification"), ("cvt", "CvtForImageClassification"), ("data2vec-vision", "Data2VecVisionForImageClassification"), ("deit", ("DeiTForImageClassification", "DeiTForImageClassificationWithTeacher")), @@ -938,6 +940,7 @@ # Backbone mapping ("bit", "BitBackbone"), ("convnext", "ConvNextBackbone"), + ("convnextv2", "ConvNextV2Backbone"), ("dinat", "DinatBackbone"), ("maskformer-swin", "MaskFormerSwinBackbone"), ("nat", "NatBackbone"), diff --git a/src/transformers/models/convnext/image_processing_convnext.py b/src/transformers/models/convnext/image_processing_convnext.py index 6d1892ef87fe..5e4d30c5c9e8 100644 --- a/src/transformers/models/convnext/image_processing_convnext.py +++ b/src/transformers/models/convnext/image_processing_convnext.py @@ -61,7 +61,7 @@ class ConvNextImageProcessor(BaseImageProcessor): be matched to `int(size["shortest_edge"]/crop_pct)`, after which the image is cropped to `(size["shortest_edge"], size["shortest_edge"])`. Only has an effect if `do_resize` is set to `True`. Can be overriden by `size` in the `preprocess` method. - crop_pct (`float` *optional*, defaults to 244 / 256): + crop_pct (`float` *optional*, defaults to 224 / 256): Percentage of the image to crop. Only has an effect if `do_resize` is `True` and size < 384. Can be overriden by `crop_pct` in the `preprocess` method. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`): diff --git a/src/transformers/models/convnextv2/__init__.py b/src/transformers/models/convnextv2/__init__.py new file mode 100644 index 000000000000..9bfd6b26e05c --- /dev/null +++ b/src/transformers/models/convnextv2/__init__.py @@ -0,0 +1,73 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +# rely on isort to merge the imports +from ...utils import ( + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, +) + + +_import_structure = { + "configuration_convnextv2": [ + "CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP", + "ConvNextV2Config", + ] +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_convnextv2"] = [ + "CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST", + "ConvNextV2ForImageClassification", + "ConvNextV2Model", + "ConvNextV2PreTrainedModel", + "ConvNextV2Backbone", + ] + + +if TYPE_CHECKING: + from .configuration_convnextv2 import ( + CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP, + ConvNextV2Config, + ) + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_convnextv2 import ( + CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST, + ConvNextV2Backbone, + ConvNextV2ForImageClassification, + ConvNextV2Model, + ConvNextV2PreTrainedModel, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure) diff --git a/src/transformers/models/convnextv2/configuration_convnextv2.py b/src/transformers/models/convnextv2/configuration_convnextv2.py new file mode 100644 index 000000000000..4c79d59b104b --- /dev/null +++ b/src/transformers/models/convnextv2/configuration_convnextv2.py @@ -0,0 +1,114 @@ +# coding=utf-8 +# Copyright 2023 Meta Platforms, Inc. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ConvNeXTV2 model configuration""" + + +from ...configuration_utils import PretrainedConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "facebook/convnextv2-tiny-1k-224": "https://huggingface.co/facebook/convnextv2-tiny-1k-224/resolve/main/config.json", +} + + +class ConvNextV2Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`ConvNextV2Model`]. It is used to instantiate an + ConvNeXTV2 model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the ConvNeXTV2 + [facebook/convnextv2-tiny-1k-224](https://huggingface.co/facebook/convnextv2-tiny-1k-224) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + num_channels (`int`, *optional*, defaults to 3): + The number of input channels. + patch_size (`int`, optional, defaults to 4): + Patch size to use in the patch embedding layer. + num_stages (`int`, optional, defaults to 4): + The number of stages in the model. + hidden_sizes (`List[int]`, *optional*, defaults to `[96, 192, 384, 768]`): + Dimensionality (hidden size) at each stage. + depths (`List[int]`, *optional*, defaults to `[3, 3, 9, 3]`): + Depth (number of blocks) for each stage. + hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in each block. If string, `"gelu"`, `"relu"`, + `"selu"` and `"gelu_new"` are supported. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the layer normalization layers. + drop_path_rate (`float`, *optional*, defaults to 0.0): + The drop rate for stochastic depth. + out_features (`List[str]`, *optional*): + If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. + (depending on how many stages the model has). Will default to the last stage if unset. + + Example: + ```python + >>> from transformers import ConvNeXTV2Config, ConvNextV2Model + + >>> # Initializing a ConvNeXTV2 convnextv2-tiny-1k-224 style configuration + >>> configuration = ConvNeXTV2Config() + + >>> # Initializing a model (with random weights) from the convnextv2-tiny-1k-224 style configuration + >>> model = ConvNextV2Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "convnextv2" + + def __init__( + self, + num_channels=3, + patch_size=4, + num_stages=4, + hidden_sizes=None, + depths=None, + hidden_act="gelu", + initializer_range=0.02, + layer_norm_eps=1e-12, + drop_path_rate=0.0, + image_size=224, + out_features=None, + **kwargs, + ): + super().__init__(**kwargs) + + self.num_channels = num_channels + self.patch_size = patch_size + self.num_stages = num_stages + self.hidden_sizes = [96, 192, 384, 768] if hidden_sizes is None else hidden_sizes + self.depths = [3, 3, 9, 3] if depths is None else depths + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.drop_path_rate = drop_path_rate + self.image_size = image_size + self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(self.depths) + 1)] + if out_features is not None: + if not isinstance(out_features, list): + raise ValueError("out_features should be a list") + for feature in out_features: + if feature not in self.stage_names: + raise ValueError( + f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" + ) + self.out_features = out_features diff --git a/src/transformers/models/convnextv2/convert_convnextv2_to_pytorch.py b/src/transformers/models/convnextv2/convert_convnextv2_to_pytorch.py new file mode 100644 index 000000000000..b2a0a52d27e8 --- /dev/null +++ b/src/transformers/models/convnextv2/convert_convnextv2_to_pytorch.py @@ -0,0 +1,282 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert ConvNeXTV2 checkpoints from the original repository. + +URL: https://github.com/facebookresearch/ConvNeXt""" + +import argparse +import json +import os + +import requests +import torch +from huggingface_hub import hf_hub_download +from PIL import Image + +from transformers import ConvNextImageProcessor, ConvNextV2Config, ConvNextV2ForImageClassification +from transformers.image_utils import PILImageResampling +from transformers.utils import logging + + +logging.set_verbosity_info() +logger = logging.get_logger(__name__) + + +def get_convnextv2_config(checkpoint_url): + config = ConvNextV2Config() + + if "atto" in checkpoint_url: + depths = [2, 2, 6, 2] + hidden_sizes = [40, 80, 160, 320] + if "femto" in checkpoint_url: + depths = [2, 2, 6, 2] + hidden_sizes = [48, 96, 192, 384] + if "pico" in checkpoint_url: + depths = [2, 2, 6, 2] + hidden_sizes = [64, 128, 256, 512] + if "nano" in checkpoint_url: + depths = [2, 2, 8, 2] + hidden_sizes = [80, 160, 320, 640] + if "tiny" in checkpoint_url: + depths = [3, 3, 9, 3] + hidden_sizes = [96, 192, 384, 768] + if "base" in checkpoint_url: + depths = [3, 3, 27, 3] + hidden_sizes = [128, 256, 512, 1024] + if "large" in checkpoint_url: + depths = [3, 3, 27, 3] + hidden_sizes = [192, 384, 768, 1536] + if "huge" in checkpoint_url: + depths = [3, 3, 27, 3] + hidden_sizes = [352, 704, 1408, 2816] + + num_labels = 1000 + filename = "imagenet-1k-id2label.json" + expected_shape = (1, 1000) + + repo_id = "huggingface/label-files" + config.num_labels = num_labels + id2label = json.load(open(hf_hub_download(repo_id, filename, repo_type="dataset"), "r")) + id2label = {int(k): v for k, v in id2label.items()} + + config.id2label = id2label + config.label2id = {v: k for k, v in id2label.items()} + config.hidden_sizes = hidden_sizes + config.depths = depths + + return config, expected_shape + + +def rename_key(name): + if "downsample_layers.0.0" in name: + name = name.replace("downsample_layers.0.0", "embeddings.patch_embeddings") + if "downsample_layers.0.1" in name: + name = name.replace("downsample_layers.0.1", "embeddings.norm") # we rename to layernorm later on + if "downsample_layers.1.0" in name: + name = name.replace("downsample_layers.1.0", "stages.1.downsampling_layer.0") + if "downsample_layers.1.1" in name: + name = name.replace("downsample_layers.1.1", "stages.1.downsampling_layer.1") + if "downsample_layers.2.0" in name: + name = name.replace("downsample_layers.2.0", "stages.2.downsampling_layer.0") + if "downsample_layers.2.1" in name: + name = name.replace("downsample_layers.2.1", "stages.2.downsampling_layer.1") + if "downsample_layers.3.0" in name: + name = name.replace("downsample_layers.3.0", "stages.3.downsampling_layer.0") + if "downsample_layers.3.1" in name: + name = name.replace("downsample_layers.3.1", "stages.3.downsampling_layer.1") + if "stages" in name and "downsampling_layer" not in name: + # stages.0.0. for instance should be renamed to stages.0.layers.0. + name = name[: len("stages.0")] + ".layers" + name[len("stages.0") :] + if "stages" in name: + name = name.replace("stages", "encoder.stages") + if "norm" in name: + name = name.replace("norm", "layernorm") + if "head" in name: + name = name.replace("head", "classifier") + + return name + + +# We will verify our results on an image of cute cats +def prepare_img(): + url = "http://images.cocodataset.org/val2017/000000039769.jpg" + im = Image.open(requests.get(url, stream=True).raw) + return im + + +def convert_preprocessor(checkpoint_url): + if "224" in checkpoint_url: + size = 224 + crop_pct = 224 / 256 + elif "384" in checkpoint_url: + size = 384 + crop_pct = None + else: + size = 512 + crop_pct = None + + return ConvNextImageProcessor( + size=size, + crop_pct=crop_pct, + image_mean=[0.485, 0.456, 0.406], + image_std=[0.229, 0.224, 0.225], + resample=PILImageResampling.BICUBIC, + ) + + +@torch.no_grad() +def convert_convnextv2_checkpoint(checkpoint_url, pytorch_dump_folder_path, save_model, push_to_hub): + """ + Copy/paste/tweak model's weights to our ConvNeXTV2 structure. + """ + print("Downloading original model from checkpoint...") + # define ConvNeXTV2 configuration based on URL + config, expected_shape = get_convnextv2_config(checkpoint_url) + # load original state_dict from URL + state_dict = torch.hub.load_state_dict_from_url(checkpoint_url)["model"] + + print("Converting model parameters...") + # rename keys + for key in state_dict.copy().keys(): + val = state_dict.pop(key) + state_dict[rename_key(key)] = val + # add prefix to all keys expect classifier head + for key in state_dict.copy().keys(): + val = state_dict.pop(key) + if not key.startswith("classifier"): + key = "convnextv2." + key + state_dict[key] = val + + # load HuggingFace model + model = ConvNextV2ForImageClassification(config) + model.load_state_dict(state_dict) + model.eval() + + # Check outputs on an image, prepared by ConvNextImageProcessor + preprocessor = convert_preprocessor(checkpoint_url) + inputs = preprocessor(images=prepare_img(), return_tensors="pt") + logits = model(**inputs).logits + + # note: the logits below were obtained without center cropping + if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt": + expected_logits = torch.tensor([-0.3930, 0.1747, -0.5246, 0.4177, 0.4295]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_femto_1k_224_ema.pt": + expected_logits = torch.tensor([-0.1727, -0.5341, -0.7818, -0.4745, -0.6566]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_pico_1k_224_ema.pt": + expected_logits = torch.tensor([-0.0333, 0.1563, -0.9137, 0.1054, 0.0381]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_nano_1k_224_ema.pt": + expected_logits = torch.tensor([-0.1744, -0.1555, -0.0713, 0.0950, -0.1431]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_tiny_1k_224_ema.pt": + expected_logits = torch.tensor([0.9996, 0.1966, -0.4386, -0.3472, 0.6661]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_base_1k_224_ema.pt": + expected_logits = torch.tensor([-0.2553, -0.6708, -0.1359, 0.2518, -0.2488]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_large_1k_224_ema.pt": + expected_logits = torch.tensor([-0.0673, -0.5627, -0.3753, -0.2722, 0.0178]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_huge_1k_224_ema.pt": + expected_logits = torch.tensor([-0.6377, -0.7458, -0.2150, 0.1184, -0.0597]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_224_ema.pt": + expected_logits = torch.tensor([1.0799, 0.2322, -0.8860, 1.0219, 0.6231]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_384_ema.pt": + expected_logits = torch.tensor([0.3766, 0.4917, -1.1426, 0.9942, 0.6024]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_224_ema.pt": + expected_logits = torch.tensor([0.4220, -0.6919, -0.4317, -0.2881, -0.6609]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_384_ema.pt": + expected_logits = torch.tensor([0.1082, -0.8286, -0.5095, 0.4681, -0.8085]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_224_ema.pt": + expected_logits = torch.tensor([-0.2419, -0.6221, 0.2176, -0.0980, -0.7527]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_384_ema.pt": + expected_logits = torch.tensor([0.0391, -0.4371, 0.3786, 0.1251, -0.2784]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_224_ema.pt": + expected_logits = torch.tensor([-0.0504, 0.5636, -0.1729, -0.6507, -0.3949]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_384_ema.pt": + expected_logits = torch.tensor([0.3560, 0.9486, 0.3149, -0.2667, -0.5138]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_384_ema.pt": + expected_logits = torch.tensor([-0.2469, -0.4550, -0.5853, -0.0810, 0.0309]) + elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_512_ema.pt": + expected_logits = torch.tensor([-0.3090, 0.0802, -0.0682, -0.1979, -0.2826]) + else: + raise ValueError(f"Unknown URL: {checkpoint_url}") + + assert torch.allclose(logits[0, :5], expected_logits, atol=1e-3) + assert logits.shape == expected_shape + print("Model outputs match the original results!") + + if save_model: + print("Saving model to local...") + # Create folder to save model + if not os.path.isdir(pytorch_dump_folder_path): + os.mkdir(pytorch_dump_folder_path) + + model.save_pretrained(pytorch_dump_folder_path) + preprocessor.save_pretrained(pytorch_dump_folder_path) + + model_name = "convnextv2" + if "atto" in checkpoint_url: + model_name += "-atto" + if "femto" in checkpoint_url: + model_name += "-femto" + if "pico" in checkpoint_url: + model_name += "-pico" + if "nano" in checkpoint_url: + model_name += "-nano" + elif "tiny" in checkpoint_url: + model_name += "-tiny" + elif "base" in checkpoint_url: + model_name += "-base" + elif "large" in checkpoint_url: + model_name += "-large" + elif "huge" in checkpoint_url: + model_name += "-huge" + if "22k" in checkpoint_url and "1k" not in checkpoint_url: + model_name += "-22k" + elif "22k" in checkpoint_url and "1k" in checkpoint_url: + model_name += "-22k-1k" + elif "1k" in checkpoint_url: + model_name += "-1k" + if "224" in checkpoint_url: + model_name += "-224" + elif "384" in checkpoint_url: + model_name += "-384" + elif "512" in checkpoint_url: + model_name += "-512" + + if push_to_hub: + print(f"Pushing {model_name} to the hub...") + model.push_to_hub(model_name) + preprocessor.push_to_hub(model_name) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # Required parameters + parser.add_argument( + "--checkpoint_url", + default="https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt", + type=str, + help="URL of the original ConvNeXTV2 checkpoint you'd like to convert.", + ) + parser.add_argument( + "--pytorch_dump_folder_path", + default="model", + type=str, + help="Path to the output PyTorch model directory.", + ) + parser.add_argument("--save_model", action="store_true", help="Save model to local") + parser.add_argument("--push_to_hub", action="store_true", help="Push model and image preprocessor to the hub") + + args = parser.parse_args() + convert_convnextv2_checkpoint( + args.checkpoint_url, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub + ) diff --git a/src/transformers/models/convnextv2/modeling_convnextv2.py b/src/transformers/models/convnextv2/modeling_convnextv2.py new file mode 100644 index 000000000000..3ee1b34f385e --- /dev/null +++ b/src/transformers/models/convnextv2/modeling_convnextv2.py @@ -0,0 +1,593 @@ +# coding=utf-8 +# Copyright 2023 Meta Platforms, Inc. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch ConvNextV2 model.""" + + +from typing import Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from ...activations import ACT2FN +from ...modeling_outputs import ( + BackboneOutput, + BaseModelOutputWithNoAttention, + BaseModelOutputWithPoolingAndNoAttention, + ImageClassifierOutputWithNoAttention, +) +from ...modeling_utils import BackboneMixin, PreTrainedModel +from ...utils import ( + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_convnextv2 import ConvNextV2Config + + +logger = logging.get_logger(__name__) + +# General docstring +_CONFIG_FOR_DOC = "ConvNextV2Config" + +# Base docstring +_CHECKPOINT_FOR_DOC = "facebook/convnextv2-tiny-1k-224" +_EXPECTED_OUTPUT_SHAPE = [1, 768, 7, 7] + +# Image classification docstring +_IMAGE_CLASS_CHECKPOINT = "facebook/convnextv2-tiny-1k-224" +_IMAGE_CLASS_EXPECTED_OUTPUT = "tabby, tabby cat" + +CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "facebook/convnextv2-tiny-1k-224", + # See all ConvNextV2 models at https://huggingface.co/models?filter=convnextv2 +] + + +# Copied from transformers.models.beit.modeling_beit.drop_path +def drop_path(input, drop_prob: float = 0.0, training: bool = False): + """ + Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + + Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks, + however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the + layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the + argument. + """ + if drop_prob == 0.0 or not training: + return input + keep_prob = 1 - drop_prob + shape = (input.shape[0],) + (1,) * (input.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=input.dtype, device=input.device) + random_tensor.floor_() # binarize + output = input.div(keep_prob) * random_tensor + return output + + +# Copied from transformers.models.beit.modeling_beit.BeitDropPath with Beit->ConvNextV2 +class ConvNextV2DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob: Optional[float] = None) -> None: + super().__init__() + self.drop_prob = drop_prob + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return drop_path(hidden_states, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return "p={}".format(self.drop_prob) + + +class ConvNextV2GRN(nn.Module): + """GRN (Global Response Normalization) layer""" + + def __init__(self, dim: int): + super().__init__() + self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim)) + self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim)) + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + # Compute and normalize global spatial feature maps + global_features = torch.norm(hidden_states, p=2, dim=(1, 2), keepdim=True) + norm_features = global_features / (global_features.mean(dim=-1, keepdim=True) + 1e-6) + hidden_states = self.gamma * (hidden_states * norm_features) + self.beta + hidden_states + + return hidden_states + + +# Copied from transformers.models.convnext.modeling_convnext.ConvNextLayerNorm with ConvNext->ConvNextV2 +class ConvNextV2LayerNorm(nn.Module): + r"""LayerNorm that supports two data formats: channels_last (default) or channels_first. + The ordering of the dimensions in the inputs. channels_last corresponds to inputs with shape (batch_size, height, + width, channels) while channels_first corresponds to inputs with shape (batch_size, channels, height, width). + """ + + def __init__(self, normalized_shape, eps=1e-6, data_format="channels_last"): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.data_format = data_format + if self.data_format not in ["channels_last", "channels_first"]: + raise NotImplementedError(f"Unsupported data format: {self.data_format}") + self.normalized_shape = (normalized_shape,) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.data_format == "channels_last": + x = torch.nn.functional.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps) + elif self.data_format == "channels_first": + input_dtype = x.dtype + x = x.float() + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = x.to(dtype=input_dtype) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x + + +# Copied from transformers.models.convnext.modeling_convnext.ConvNextEmbeddings with ConvNext->ConvNextV2 +class ConvNextV2Embeddings(nn.Module): + """This class is comparable to (and inspired by) the SwinEmbeddings class + found in src/transformers/models/swin/modeling_swin.py. + """ + + def __init__(self, config): + super().__init__() + self.patch_embeddings = nn.Conv2d( + config.num_channels, config.hidden_sizes[0], kernel_size=config.patch_size, stride=config.patch_size + ) + self.layernorm = ConvNextV2LayerNorm(config.hidden_sizes[0], eps=1e-6, data_format="channels_first") + self.num_channels = config.num_channels + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + num_channels = pixel_values.shape[1] + if num_channels != self.num_channels: + raise ValueError( + "Make sure that the channel dimension of the pixel values match with the one set in the configuration." + ) + embeddings = self.patch_embeddings(pixel_values) + embeddings = self.layernorm(embeddings) + return embeddings + + +class ConvNextV2Layer(nn.Module): + """This corresponds to the `Block` class in the original implementation. + + There are two equivalent implementations: [DwConv, LayerNorm (channels_first), Conv, GELU,1x1 Conv]; all in (N, C, + H, W) (2) [DwConv, Permute to (N, H, W, C), LayerNorm (channels_last), Linear, GELU, Linear]; Permute back + + The authors used (2) as they find it slightly faster in PyTorch. + + Args: + config ([`ConvNextV2Config`]): Model configuration class. + dim (`int`): Number of input channels. + drop_path (`float`): Stochastic depth rate. Default: 0.0. + """ + + def __init__(self, config, dim, drop_path=0): + super().__init__() + # depthwise conv + self.dwconv = nn.Conv2d(dim, dim, kernel_size=7, padding=3, groups=dim) + self.layernorm = ConvNextV2LayerNorm(dim, eps=1e-6) + # pointwise/1x1 convs, implemented with linear layers + self.pwconv1 = nn.Linear(dim, 4 * dim) + self.act = ACT2FN[config.hidden_act] + self.grn = ConvNextV2GRN(4 * dim) + self.pwconv2 = nn.Linear(4 * dim, dim) + self.drop_path = ConvNextV2DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + def forward(self, hidden_states: torch.FloatTensor) -> torch.Tensor: + input = hidden_states + x = self.dwconv(hidden_states) + # (batch_size, num_channels, height, width) -> (batch_size, height, width, num_channels) + x = x.permute(0, 2, 3, 1) + x = self.layernorm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.grn(x) + x = self.pwconv2(x) + # (batch_size, height, width, num_channels) -> (batch_size, num_channels, height, width) + x = x.permute(0, 3, 1, 2) + + x = input + self.drop_path(x) + return x + + +# Copied from transformers.models.convnext.modeling_convnext.ConvNextStage with ConvNeXT->ConvNeXTV2, ConvNext->ConvNextV2 +class ConvNextV2Stage(nn.Module): + """ConvNeXTV2 stage, consisting of an optional downsampling layer + multiple residual blocks. + + Args: + config ([`ConvNextV2Config`]): Model configuration class. + in_channels (`int`): Number of input channels. + out_channels (`int`): Number of output channels. + depth (`int`): Number of residual blocks. + drop_path_rates(`List[float]`): Stochastic depth rates for each layer. + """ + + def __init__(self, config, in_channels, out_channels, kernel_size=2, stride=2, depth=2, drop_path_rates=None): + super().__init__() + + if in_channels != out_channels or stride > 1: + self.downsampling_layer = nn.Sequential( + ConvNextV2LayerNorm(in_channels, eps=1e-6, data_format="channels_first"), + nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride), + ) + else: + self.downsampling_layer = nn.Identity() + drop_path_rates = drop_path_rates or [0.0] * depth + self.layers = nn.Sequential( + *[ConvNextV2Layer(config, dim=out_channels, drop_path=drop_path_rates[j]) for j in range(depth)] + ) + + def forward(self, hidden_states: torch.FloatTensor) -> torch.Tensor: + hidden_states = self.downsampling_layer(hidden_states) + hidden_states = self.layers(hidden_states) + return hidden_states + + +# Copied from transformers.models.convnext.modeling_convnext.ConvNextEncoder with ConvNext->ConvNextV2 +class ConvNextV2Encoder(nn.Module): + def __init__(self, config): + super().__init__() + self.stages = nn.ModuleList() + drop_path_rates = [ + x.tolist() for x in torch.linspace(0, config.drop_path_rate, sum(config.depths)).split(config.depths) + ] + prev_chs = config.hidden_sizes[0] + for i in range(config.num_stages): + out_chs = config.hidden_sizes[i] + stage = ConvNextV2Stage( + config, + in_channels=prev_chs, + out_channels=out_chs, + stride=2 if i > 0 else 1, + depth=config.depths[i], + drop_path_rates=drop_path_rates[i], + ) + self.stages.append(stage) + prev_chs = out_chs + + def forward( + self, + hidden_states: torch.FloatTensor, + output_hidden_states: Optional[bool] = False, + return_dict: Optional[bool] = True, + ) -> Union[Tuple, BaseModelOutputWithNoAttention]: + all_hidden_states = () if output_hidden_states else None + + for i, layer_module in enumerate(self.stages): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + hidden_states = layer_module(hidden_states) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, all_hidden_states] if v is not None) + + return BaseModelOutputWithNoAttention( + last_hidden_state=hidden_states, + hidden_states=all_hidden_states, + ) + + +# Copied from transformers.models.convnext.modeling_convnext.ConvNextPreTrainedModel with ConvNext->ConvNextV2, convnext->convnextv2 +class ConvNextV2PreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = ConvNextV2Config + base_model_prefix = "convnextv2" + main_input_name = "pixel_values" + supports_gradient_checkpointing = True + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, (nn.Linear, nn.Conv2d)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, ConvNextV2Encoder): + module.gradient_checkpointing = value + + +CONVNEXTV2_START_DOCSTRING = r""" + This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it + as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and + behavior. + + Parameters: + config ([`ConvNextV2Config`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +CONVNEXTV2_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`ConvNextImageProcessor`]. See + [`ConvNextImageProcessor.__call__`] for details. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare ConvNextV2 model outputting raw features without any specific head on top.", + CONVNEXTV2_START_DOCSTRING, +) +# Copied from transformers.models.convnext.modeling_convnext.ConvNextModel with CONVNEXT->CONVNEXTV2, ConvNext->ConvNextV2 +class ConvNextV2Model(ConvNextV2PreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.config = config + + self.embeddings = ConvNextV2Embeddings(config) + self.encoder = ConvNextV2Encoder(config) + + # final layernorm layer + self.layernorm = nn.LayerNorm(config.hidden_sizes[-1], eps=config.layer_norm_eps) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(CONVNEXTV2_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=BaseModelOutputWithPoolingAndNoAttention, + config_class=_CONFIG_FOR_DOC, + modality="vision", + expected_output=_EXPECTED_OUTPUT_SHAPE, + ) + def forward( + self, + pixel_values: torch.FloatTensor = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPoolingAndNoAttention]: + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + embedding_output = self.embeddings(pixel_values) + + encoder_outputs = self.encoder( + embedding_output, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + + # global average pooling, (N, C, H, W) -> (N, C) + pooled_output = self.layernorm(last_hidden_state.mean([-2, -1])) + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndNoAttention( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + ) + + +@add_start_docstrings( + """ + ConvNextV2 Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for + ImageNet. + """, + CONVNEXTV2_START_DOCSTRING, +) +# Copied from transformers.models.convnext.modeling_convnext.ConvNextForImageClassification with CONVNEXT->CONVNEXTV2,ConvNext->ConvNextV2,convnext->convnextv2 +class ConvNextV2ForImageClassification(ConvNextV2PreTrainedModel): + def __init__(self, config): + super().__init__(config) + + self.num_labels = config.num_labels + self.convnextv2 = ConvNextV2Model(config) + + # Classifier head + self.classifier = ( + nn.Linear(config.hidden_sizes[-1], config.num_labels) if config.num_labels > 0 else nn.Identity() + ) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(CONVNEXTV2_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_IMAGE_CLASS_CHECKPOINT, + output_type=ImageClassifierOutputWithNoAttention, + config_class=_CONFIG_FOR_DOC, + expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT, + ) + def forward( + self, + pixel_values: torch.FloatTensor = None, + labels: Optional[torch.LongTensor] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, ImageClassifierOutputWithNoAttention]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the image classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.convnextv2(pixel_values, output_hidden_states=output_hidden_states, return_dict=return_dict) + + pooled_output = outputs.pooler_output if return_dict else outputs[1] + + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return ImageClassifierOutputWithNoAttention( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + ) + + +@add_start_docstrings( + """ + ConvNeXT V2 backbone, to be used with frameworks like DETR and MaskFormer. + """, + CONVNEXTV2_START_DOCSTRING, +) +# Copied from transformers.models.convnext.modeling_convnext.ConvNextBackbone with CONVNEXT->CONVNEXTV2,ConvNext->ConvNextV2,facebook/convnext-tiny-224->facebook/convnextv2-tiny-1k-224 +class ConvNextV2Backbone(ConvNextV2PreTrainedModel, BackboneMixin): + def __init__(self, config): + super().__init__(config) + + self.stage_names = config.stage_names + self.embeddings = ConvNextV2Embeddings(config) + self.encoder = ConvNextV2Encoder(config) + + self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] + + out_feature_channels = {} + out_feature_channels["stem"] = config.hidden_sizes[0] + for idx, stage in enumerate(self.stage_names[1:]): + out_feature_channels[stage] = config.hidden_sizes[idx] + + self.out_feature_channels = out_feature_channels + + # Add layer norms to hidden states of out_features + hidden_states_norms = {} + for stage, num_channels in zip(self.out_features, self.channels): + hidden_states_norms[stage] = ConvNextV2LayerNorm(num_channels, data_format="channels_first") + self.hidden_states_norms = nn.ModuleDict(hidden_states_norms) + + # initialize weights and apply final processing + self.post_init() + + @property + def channels(self): + return [self.out_feature_channels[name] for name in self.out_features] + + @add_start_docstrings_to_model_forward(CONVNEXTV2_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + pixel_values: torch.Tensor, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> BackboneOutput: + """ + Returns: + + Examples: + + ```python + >>> from transformers import AutoImageProcessor, AutoBackbone + >>> import torch + >>> from PIL import Image + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> processor = AutoImageProcessor.from_pretrained("facebook/convnextv2-tiny-1k-224") + >>> model = AutoBackbone.from_pretrained("facebook/convnextv2-tiny-1k-224") + + >>> inputs = processor(image, return_tensors="pt") + >>> outputs = model(**inputs) + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + embedding_output = self.embeddings(pixel_values) + + outputs = self.encoder( + embedding_output, + output_hidden_states=True, + return_dict=True, + ) + + hidden_states = outputs.hidden_states + + feature_maps = () + # we skip the stem + for idx, (stage, hidden_state) in enumerate(zip(self.stage_names[1:], hidden_states[1:])): + if stage in self.out_features: + hidden_state = self.hidden_states_norms[stage](hidden_state) + feature_maps += (hidden_state,) + + if not return_dict: + output = (feature_maps,) + if output_hidden_states: + output += (outputs.hidden_states,) + return output + + return BackboneOutput( + feature_maps=feature_maps, + hidden_states=outputs.hidden_states if output_hidden_states else None, + attentions=None, + ) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 1f5931885f83..fc9a8d324a61 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1803,6 +1803,37 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class ConvNextV2Backbone(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class ConvNextV2ForImageClassification(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class ConvNextV2Model(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class ConvNextV2PreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + CTRL_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/tests/models/convnextv2/__init__.py b/tests/models/convnextv2/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/convnextv2/test_modeling_convnextv2.py b/tests/models/convnextv2/test_modeling_convnextv2.py new file mode 100644 index 000000000000..10ae34c22d83 --- /dev/null +++ b/tests/models/convnextv2/test_modeling_convnextv2.py @@ -0,0 +1,348 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch ConvNextV2 model. """ + + +import inspect +import unittest + +from transformers import ConvNextV2Config +from transformers.models.auto import get_values +from transformers.models.auto.modeling_auto import MODEL_FOR_BACKBONE_MAPPING_NAMES, MODEL_MAPPING_NAMES +from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.utils import cached_property, is_torch_available, is_vision_available + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor + + +if is_torch_available(): + import torch + + from transformers import ConvNextV2Backbone, ConvNextV2ForImageClassification, ConvNextV2Model + from transformers.models.convnextv2.modeling_convnextv2 import CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST + + +if is_vision_available(): + from PIL import Image + + from transformers import AutoImageProcessor + + +class ConvNextV2ModelTester: + def __init__( + self, + parent, + batch_size=13, + image_size=32, + num_channels=3, + num_stages=4, + hidden_sizes=[10, 20, 30, 40], + depths=[2, 2, 3, 2], + is_training=True, + use_labels=True, + intermediate_size=37, + hidden_act="gelu", + num_labels=10, + initializer_range=0.02, + out_features=["stage2", "stage3", "stage4"], + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.image_size = image_size + self.num_channels = num_channels + self.num_stages = num_stages + self.hidden_sizes = hidden_sizes + self.depths = depths + self.is_training = is_training + self.use_labels = use_labels + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.num_labels = num_labels + self.initializer_range = initializer_range + self.out_features = out_features + self.scope = scope + + def prepare_config_and_inputs(self): + pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) + + labels = None + if self.use_labels: + labels = ids_tensor([self.batch_size], self.num_labels) + + config = self.get_config() + + return config, pixel_values, labels + + def get_config(self): + return ConvNextV2Config( + num_channels=self.num_channels, + hidden_sizes=self.hidden_sizes, + depths=self.depths, + num_stages=self.num_stages, + hidden_act=self.hidden_act, + is_decoder=False, + initializer_range=self.initializer_range, + out_features=self.out_features, + num_labels=self.num_labels, + ) + + def create_and_check_model(self, config, pixel_values, labels): + model = ConvNextV2Model(config=config) + model.to(torch_device) + model.eval() + result = model(pixel_values) + # expected last hidden states: B, C, H // 32, W // 32 + self.parent.assertEqual( + result.last_hidden_state.shape, + (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32), + ) + + def create_and_check_for_image_classification(self, config, pixel_values, labels): + model = ConvNextV2ForImageClassification(config) + model.to(torch_device) + model.eval() + result = model(pixel_values, labels=labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels)) + + def create_and_check_backbone(self, config, pixel_values, labels): + model = ConvNextV2Backbone(config=config) + model.to(torch_device) + model.eval() + result = model(pixel_values) + + # verify hidden states + self.parent.assertEqual(len(result.feature_maps), len(config.out_features)) + self.parent.assertListEqual(list(result.feature_maps[0].shape), [self.batch_size, self.hidden_sizes[1], 4, 4]) + + # verify channels + self.parent.assertEqual(len(model.channels), len(config.out_features)) + self.parent.assertListEqual(model.channels, config.hidden_sizes[1:]) + + # verify backbone works with out_features=None + config.out_features = None + model = ConvNextV2Backbone(config=config) + model.to(torch_device) + model.eval() + result = model(pixel_values) + + # verify feature maps + self.parent.assertEqual(len(result.feature_maps), 1) + self.parent.assertListEqual(list(result.feature_maps[0].shape), [self.batch_size, self.hidden_sizes[-1], 1, 1]) + + # verify channels + self.parent.assertEqual(len(model.channels), 1) + self.parent.assertListEqual(model.channels, [config.hidden_sizes[-1]]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, pixel_values, labels = config_and_inputs + inputs_dict = {"pixel_values": pixel_values} + return config, inputs_dict + + def prepare_config_and_inputs_with_labels(self): + config_and_inputs = self.prepare_config_and_inputs() + config, pixel_values, labels = config_and_inputs + inputs_dict = {"pixel_values": pixel_values, "labels": labels} + return config, inputs_dict + + +@require_torch +class ConvNextV2ModelTest(ModelTesterMixin, unittest.TestCase): + """ + Here we also overwrite some of the tests of test_modeling_common.py, as ConvNextV2 does not use input_ids, inputs_embeds, + attention_mask and seq_length. + """ + + all_model_classes = ( + ( + ConvNextV2Model, + ConvNextV2ForImageClassification, + ConvNextV2Backbone, + ) + if is_torch_available() + else () + ) + + fx_compatible = False + test_pruning = False + test_resize_embeddings = False + test_head_masking = False + has_attentions = False + + def setUp(self): + self.model_tester = ConvNextV2ModelTester(self) + self.config_tester = ConfigTester(self, config_class=ConvNextV2Config, has_text_modality=False, hidden_size=37) + + def test_config(self): + self.create_and_test_config_common_properties() + self.config_tester.create_and_test_config_to_json_string() + self.config_tester.create_and_test_config_to_json_file() + self.config_tester.create_and_test_config_from_and_save_pretrained() + self.config_tester.create_and_test_config_with_num_labels() + self.config_tester.check_config_can_be_init_without_params() + self.config_tester.check_config_arguments_init() + + def create_and_test_config_common_properties(self): + return + + @unittest.skip(reason="ConvNextV2 does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="ConvNextV2 does not support input and output embeddings") + def test_model_common_attributes(self): + pass + + @unittest.skip(reason="ConvNextV2 does not use feedforward chunking") + def test_feed_forward_chunking(self): + pass + + def test_training(self): + if not self.model_tester.is_training: + return + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_with_labels() + config.return_dict = True + + if model_class.__name__ in [ + *get_values(MODEL_MAPPING_NAMES), + *get_values(MODEL_FOR_BACKBONE_MAPPING_NAMES), + ]: + continue + + model = model_class(config) + model.to(torch_device) + model.train() + inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + loss = model(**inputs).loss + loss.backward() + + def test_training_gradient_checkpointing(self): + if not self.model_tester.is_training: + return + + for model_class in self.all_model_classes: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_with_labels() + config.use_cache = False + config.return_dict = True + + if ( + model_class.__name__ + in [*get_values(MODEL_MAPPING_NAMES), *get_values(MODEL_FOR_BACKBONE_MAPPING_NAMES)] + or not model_class.supports_gradient_checkpointing + ): + continue + + model = model_class(config) + model.to(torch_device) + model.gradient_checkpointing_enable() + model.train() + inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + loss = model(**inputs).loss + loss.backward() + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["pixel_values"] + self.assertListEqual(arg_names[:1], expected_arg_names) + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_hidden_states_output(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(config) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + + hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + + expected_num_stages = self.model_tester.num_stages + self.assertEqual(len(hidden_states), expected_num_stages + 1) + + # ConvNextV2's feature maps are of shape (batch_size, num_channels, height, width) + self.assertListEqual( + list(hidden_states[0].shape[-2:]), + [self.model_tester.image_size // 4, self.model_tester.image_size // 4], + ) + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + inputs_dict["output_hidden_states"] = True + check_hidden_states_output(inputs_dict, config, model_class) + + # check that output_hidden_states also work using config + del inputs_dict["output_hidden_states"] + config.output_hidden_states = True + + check_hidden_states_output(inputs_dict, config, model_class) + + def test_for_image_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_image_classification(*config_and_inputs) + + @slow + def test_model_from_pretrained(self): + for model_name in CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = ConvNextV2Model.from_pretrained(model_name) + self.assertIsNotNone(model) + + +# We will verify our results on an image of cute cats +def prepare_img(): + image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") + return image + + +@require_torch +@require_vision +class ConvNextV2ModelIntegrationTest(unittest.TestCase): + @cached_property + def default_image_processor(self): + return AutoImageProcessor.from_pretrained("facebook/convnextv2-tiny-1k-224") if is_vision_available() else None + + @slow + def test_inference_image_classification_head(self): + model = ConvNextV2ForImageClassification.from_pretrained("facebook/convnextv2-tiny-1k-224").to(torch_device) + + preprocessor = self.default_image_processor + image = prepare_img() + inputs = preprocessor(images=image, return_tensors="pt").to(torch_device) + + # forward pass + with torch.no_grad(): + outputs = model(**inputs) + + # verify the logits + expected_shape = torch.Size((1, 1000)) + self.assertEqual(outputs.logits.shape, expected_shape) + + expected_slice = torch.tensor([-0.3083, -0.3040, -0.4344]).to(torch_device) + self.assertTrue(torch.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4)) diff --git a/utils/check_repo.py b/utils/check_repo.py index fca0cd86ef90..c59897d832b5 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -820,6 +820,7 @@ def find_all_documented_objects(): "AutoBackbone", "BitBackbone", "ConvNextBackbone", + "ConvNextV2Backbone", "DinatBackbone", "MaskFormerSwinBackbone", "MaskFormerSwinConfig", From 6d0b040f66a2b915497a4b88033e410744f2731d Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 14 Mar 2023 10:13:16 +0100 Subject: [PATCH 092/277] Update 2 doctest expected values for torch 2.0.0 (#22148) update values Co-authored-by: ydshieh --- docs/source/en/pipeline_tutorial.mdx | 2 +- src/transformers/generation/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/pipeline_tutorial.mdx b/docs/source/en/pipeline_tutorial.mdx index 00dceeb4f243..873d497d3ef9 100644 --- a/docs/source/en/pipeline_tutorial.mdx +++ b/docs/source/en/pipeline_tutorial.mdx @@ -247,7 +247,7 @@ For example, if you use this [invoice image](https://huggingface.co/spaces/impir ... image="https://huggingface.co/spaces/impira/docquery/resolve/2359223c1837a7587402bda0f2643382a6eefeab/invoice.png", ... question="What is the invoice number?", ... ) -[{'score': 0.42514941096305847, 'answer': 'us-001', 'start': 16, 'end': 16}] +[{'score': 0.42515, 'answer': 'us-001', 'start': 16, 'end': 16}] ``` diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 7091b49e3ecc..ad18644ee5cb 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -2401,7 +2401,7 @@ def sample( ... ) >>> tokenizer.batch_decode(outputs, skip_special_tokens=True) - ['Today is a beautiful day, and a wonderful day.\n\nI was lucky enough to meet the'] + ['Today is a beautiful day, and we must do everything possible to make it a day of celebration.'] ```""" # init values logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList() From 4501ebe921d9e6c90f64c5995cd7aeaeb53789b6 Mon Sep 17 00:00:00 2001 From: Nicola Procopio Date: Tue, 14 Mar 2023 12:09:36 +0100 Subject: [PATCH 093/277] Translation Italian: perf_train_cpu and perf_train_cpu_many (#22151) * added translated files added perf_train_cpu and perf_train_cpu_many * updated toctree --- docs/source/it/_toctree.yml | 4 + docs/source/it/perf_train_cpu.mdx | 65 ++++++++++++ docs/source/it/perf_train_cpu_many.mdx | 137 +++++++++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 docs/source/it/perf_train_cpu.mdx create mode 100644 docs/source/it/perf_train_cpu_many.mdx diff --git a/docs/source/it/_toctree.yml b/docs/source/it/_toctree.yml index 86c3465b478c..c0f4fd082491 100644 --- a/docs/source/it/_toctree.yml +++ b/docs/source/it/_toctree.yml @@ -33,6 +33,10 @@ title: Convertire modelli tensorflow - local: serialization title: Esporta modelli Transformers + - local: perf_train_cpu + title: Addestramento efficiente su CPU + - local: perf_train_cpu_many + title: Addestramento efficiente su multiple CPU - local: big_models title: Istanziare un big model - local: debugging diff --git a/docs/source/it/perf_train_cpu.mdx b/docs/source/it/perf_train_cpu.mdx new file mode 100644 index 000000000000..bf9b265ba301 --- /dev/null +++ b/docs/source/it/perf_train_cpu.mdx @@ -0,0 +1,65 @@ + + +# Addestramento efficiente su CPU + +Questa guida si concentra su come addestrare in maniera efficiente grandi modelli su CPU. + +## Mixed precision con IPEX + +IPEX è ottimizzato per CPU con AVX-512 o superiore, e funziona per le CPU con solo AVX2. Pertanto, si prevede che le prestazioni saranno più vantaggiose per le le CPU Intel con AVX-512 o superiori, mentre le CPU con solo AVX2 (ad esempio, le CPU AMD o le CPU Intel più vecchie) potrebbero ottenere prestazioni migliori con IPEX, ma non sono garantite. IPEX offre ottimizzazioni delle prestazioni per l'addestramento della CPU sia con Float32 che con BFloat16. L'uso di BFloat16 è l'argomento principale delle seguenti sezioni. + +Il tipo di dati a bassa precisione BFloat16 è stato supportato in modo nativo su 3rd Generation Xeon® Scalable Processors (aka Cooper Lake) con AVX512 e sarà supportata dalla prossima generazione di Intel® Xeon® Scalable Processors con Intel® Advanced Matrix Extensions (Intel® AMX) instruction set con prestazioni ulteriormente migliorate. L'Auto Mixed Precision per il backende della CPU è stato abilitato da PyTorch-1.10. allo stesso tempo, il supporto di Auto Mixed Precision con BFloat16 per CPU e l'ottimizzazione degli operatori BFloat16 è stata abilitata in modo massiccio in Intel® Extension per PyTorch, and parzialmente aggiornato al branch master di PyTorch. Gli utenti possono ottenere prestazioni migliori ed users experience con IPEX Auto Mixed Precision.. + +Vedi informazioni più dettagliate su [Auto Mixed Precision](https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/features/amp.html). + +### Installazione di IPEX: + +Il rilascio di IPEX segue quello di PyTorch, da installare via pip: + +| PyTorch Version | IPEX version | +| :---------------: | :----------: | +| 1.13 | 1.13.0+cpu | +| 1.12 | 1.12.300+cpu | +| 1.11 | 1.11.200+cpu | +| 1.10 | 1.10.100+cpu | + +```bash +pip install intel_extension_for_pytorch== -f https://developer.intel.com/ipex-whl-stable-cpu +``` + +Vedi altri approcci per [IPEX installation](https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/installation.html). + +### Utilizzo nel Trainer + +Per abilitare la auto mixed precision con IPEX in Trainer, l'utende dovrebbe aggiungere `use_ipex`, `bf16` e `no_cuda` negli argomenti del comando di addestramento. + +Vedi un sempio di un caso d'uso [Transformers question-answering](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering) + +- Training with IPEX using BF16 auto mixed precision on CPU: + +
 python run_qa.py \
+--model_name_or_path bert-base-uncased \
+--dataset_name squad \
+--do_train \
+--do_eval \
+--per_device_train_batch_size 12 \
+--learning_rate 3e-5 \
+--num_train_epochs 2 \
+--max_seq_length 384 \
+--doc_stride 128 \
+--output_dir /tmp/debug_squad/ \
+--use_ipex \
+--bf16 --no_cuda
+ +### Esempi pratici + +Blog: [Accelerating PyTorch Transformers with Intel Sapphire Rapids](https://huggingface.co/blog/intel-sapphire-rapids) diff --git a/docs/source/it/perf_train_cpu_many.mdx b/docs/source/it/perf_train_cpu_many.mdx new file mode 100644 index 000000000000..6fea6b85b061 --- /dev/null +++ b/docs/source/it/perf_train_cpu_many.mdx @@ -0,0 +1,137 @@ + + +# Addestramento effciente su multiple CPU + +Quando l'addestramento su una singola CPU è troppo lento, possiamo usare CPU multiple. Quasta guida si concentra su DDP basato su PyTorch abilitando l'addetramento distribuito su CPU in maniera efficiente. + +## Intel® oneCCL Bindings per PyTorch + +[Intel® oneCCL](https://github.com/oneapi-src/oneCCL) (collective communications library) è una libreria per l'addestramento efficiente del deep learning in distribuito e implementa collettivi come allreduce, allgather, alltoall. Per maggiori informazioni su oneCCL, fai riferimento a [oneCCL documentation](https://spec.oneapi.com/versions/latest/elements/oneCCL/source/index.html) e [oneCCL specification](https://spec.oneapi.com/versions/latest/elements/oneCCL/source/index.html). + +Il modulo `oneccl_bindings_for_pytorch` (`torch_ccl` precedentemente alla versione 1.12) implementa PyTorch C10D ProcessGroup API e può essere caricato dinamicamente com external ProcessGroup e funziona solo su piattaforma Linux al momento. + +Qui trovi informazioni più dettagliate per [oneccl_bind_pt](https://github.com/intel/torch-ccl). + +### Intel® oneCCL Bindings per l'installazione PyTorch: + +I file wheel sono disponibili per le seguenti versioni di Python: + +| Extension Version | Python 3.6 | Python 3.7 | Python 3.8 | Python 3.9 | Python 3.10 | +| :---------------: | :--------: | :--------: | :--------: | :--------: | :---------: | +| 1.13.0 | | √ | √ | √ | √ | +| 1.12.100 | | √ | √ | √ | √ | +| 1.12.0 | | √ | √ | √ | √ | +| 1.11.0 | | √ | √ | √ | √ | +| 1.10.0 | √ | √ | √ | √ | | + +```bash +pip install oneccl_bind_pt=={pytorch_version} -f https://developer.intel.com/ipex-whl-stable-cpu +``` + +dove `{pytorch_version}` deve essere la tua versione di PyTorch, per l'stanza 1.13.0. +Verifica altri approcci per [oneccl_bind_pt installation](https://github.com/intel/torch-ccl). +Le versioni di oneCCL e PyTorch devono combaciare. + + + +oneccl_bindings_for_pytorch 1.12.0 prebuilt wheel does not work with PyTorch 1.12.1 (it is for PyTorch 1.12.0) +PyTorch 1.12.1 should work with oneccl_bindings_for_pytorch 1.12.100 + + + +## Intel® MPI library + +Usa questa implementazione basata su standard MPI per fornire una architettura flessibile, efficiente, scalabile su cluster per Intel®. Questo componente è parte di Intel® oneAPI HPC Toolkit. + +oneccl_bindings_for_pytorch è installato insieme al set di strumenti MPI. Necessità di reperire l'ambiente prima di utilizzarlo. + +per Intel® oneCCL >= 1.12.0 + +```bash +oneccl_bindings_for_pytorch_path=$(python -c "from oneccl_bindings_for_pytorch import cwd; print(cwd)") +source $oneccl_bindings_for_pytorch_path/env/setvars.sh +``` + +per Intel® oneCCL con versione < 1.12.0 + +```bash +torch_ccl_path=$(python -c "import torch; import torch_ccl; import os; print(os.path.abspath(os.path.dirname(torch_ccl.__file__)))") +source $torch_ccl_path/env/setvars.sh +``` + +#### Installazione IPEX: + +IPEX fornisce ottimizzazioni delle prestazioni per l'addestramento della CPU sia con Float32 che con BFloat16; puoi fare riferimento a [single CPU section](./perf_train_cpu). + +Il seguente "Utilizzo in Trainer" prende come esempio mpirun nella libreria Intel® MPI. + +## Utilizzo in Trainer + +Per abilitare l'addestramento distribuito multi CPU nel Trainer con il ccl backend, gli utenti devono aggiungere **`--xpu_backend ccl`** negli argomenti del comando. + +Vediamo un esempio per il [question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering) + +Il seguente comando abilita due processi sul nodo Xeon, con un processo in esecuzione per ogni socket. Le variabili OMP_NUM_THREADS/CCL_WORKER_COUNT possono essere impostate per una prestazione ottimale. + +```shell script + export CCL_WORKER_COUNT=1 + export MASTER_ADDR=127.0.0.1 + mpirun -n 2 -genv OMP_NUM_THREADS=23 \ + python3 run_qa.py \ + --model_name_or_path bert-large-uncased \ + --dataset_name squad \ + --do_train \ + --do_eval \ + --per_device_train_batch_size 12 \ + --learning_rate 3e-5 \ + --num_train_epochs 2 \ + --max_seq_length 384 \ + --doc_stride 128 \ + --output_dir /tmp/debug_squad/ \ + --no_cuda \ + --xpu_backend ccl \ + --use_ipex +``` + +Il seguente comando abilita l'addestramento per un totale di quattro processi su due Xeon (node0 e node1, prendendo node0 come processo principale), ppn (processes per node) è impostato a 2, on un processo in esecuzione per ogni socket. Le variabili OMP_NUM_THREADS/CCL_WORKER_COUNT possono essere impostate per una prestazione ottimale. + +In node0, è necessario creare un file di configurazione che contenga gli indirizzi IP di ciascun nodo (per esempio hostfile) e passare il percorso del file di configurazione come parametro. + +```shell script + cat hostfile + xxx.xxx.xxx.xxx #node0 ip + xxx.xxx.xxx.xxx #node1 ip +``` + +A questo punto, esegui il seguente comando nel nodo0 e **4DDP** sarà abilitato in node0 e node1 con BF16 auto mixed precision: + +```shell script + export CCL_WORKER_COUNT=1 + export MASTER_ADDR=xxx.xxx.xxx.xxx #node0 ip + mpirun -f hostfile -n 4 -ppn 2 \ + -genv OMP_NUM_THREADS=23 \ + python3 run_qa.py \ + --model_name_or_path bert-large-uncased \ + --dataset_name squad \ + --do_train \ + --do_eval \ + --per_device_train_batch_size 12 \ + --learning_rate 3e-5 \ + --num_train_epochs 2 \ + --max_seq_length 384 \ + --doc_stride 128 \ + --output_dir /tmp/debug_squad/ \ + --no_cuda \ + --xpu_backend ccl \ + --use_ipex \ + --bf16 +``` From 4f9f1909fb3382765bcf3d24b7462401fbd50064 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 14 Mar 2023 09:20:16 -0400 Subject: [PATCH 094/277] Fix big model inference for T5 models in float16 (#22095) * Fix big model inference for T5 models in float16 * Apply suggestions from code review Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> * Style * Trigger CI with latest release --------- Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- src/transformers/modeling_utils.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6a76accf5691..697b3deb14d7 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2547,6 +2547,11 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P ) >= version.parse("0.37.0") if isinstance(device_map, str): + special_dtypes = { + name: torch.float32 + for name, _ in model.named_parameters() + if any(m in name for m in keep_in_fp32_modules) + } if model._no_split_modules is None: raise ValueError(f"{model.__class__.__name__} does not support `device_map='{device_map}'` yet.") no_split_modules = model._no_split_modules @@ -2557,22 +2562,25 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P ) elif device_map in ["balanced", "balanced_low_0"] and get_balanced_memory is None: raise ValueError(f"`device_map={device_map}` requires a source install of Accelerate.") + + kwargs = {"no_split_module_classes": no_split_modules, "max_memory": max_memory} + if "special_dtypes" in inspect.signature(infer_auto_device_map).parameters: + kwargs["special_dtypes"] = special_dtypes + elif len(special_dtypes) > 0: + logger.warn( + "This model has some weights that should be kept in higher precision, you need to upgrade " + "`accelerate` to properly deal with them (`pip install --upgrade accelerate`)." + ) if device_map != "sequential" and get_balanced_memory is not None: max_memory = get_balanced_memory( model, - max_memory=max_memory, - no_split_module_classes=no_split_modules, dtype=torch_dtype, low_zero=(device_map == "balanced_low_0"), + **kwargs, ) # Make sure tied weights are tied before creating the device map. model.tie_weights() - device_map = infer_auto_device_map( - model, - no_split_module_classes=no_split_modules, - dtype=torch_dtype if not load_in_8bit else torch.int8, - max_memory=max_memory, - ) + device_map = infer_auto_device_map(model, dtype=torch_dtype if not load_in_8bit else torch.int8, **kwargs) if load_in_8bit: # The LM head / tied weights or any last module can stay on disk / CPU From 2e5b6009d03ba9cda65035b7ee22e5699851fafb Mon Sep 17 00:00:00 2001 From: Alara Dirik <8944735+alaradirik@users.noreply.github.com> Date: Tue, 14 Mar 2023 16:55:18 +0300 Subject: [PATCH 095/277] Create MaskedImageCompletionOutput and fix ViT docs (#22152) * create MaskedImageCompletionOutput * fix bugs * fix bugs --- src/transformers/modeling_outputs.py | 28 +++++++++++++++++++++ src/transformers/models/vit/modeling_vit.py | 15 +++++++---- tests/models/vit/test_modeling_vit.py | 4 +-- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/transformers/modeling_outputs.py b/src/transformers/modeling_outputs.py index 4f7540d0ff9e..a4d465b1d48f 100755 --- a/src/transformers/modeling_outputs.py +++ b/src/transformers/modeling_outputs.py @@ -1281,6 +1281,34 @@ class ImageSuperResolutionOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None +@dataclass +class MaskedImageCompletionOutput(ModelOutput): + """ + Base class for outputs of masked image completion / in-painting models. + + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Reconstruction loss. + reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Reconstructed / completed images. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each stage) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states + (also called feature maps) of the model at the output of each stage. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + loss: Optional[torch.FloatTensor] = None + reconstruction: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + @dataclass class Wav2Vec2BaseModelOutput(ModelOutput): """ diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index 449eda3ee821..154afdb211f3 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -25,7 +25,12 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput, MaskedLMOutput +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPooling, + ImageClassifierOutput, + MaskedImageCompletionOutput, +) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( @@ -643,7 +648,7 @@ def __init__(self, config: ViTConfig) -> None: self.post_init() @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @replace_return_docstrings(output_type=MaskedImageCompletionOutput, config_class=_CONFIG_FOR_DOC) def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -653,7 +658,7 @@ def forward( output_hidden_states: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[tuple, MaskedLMOutput]: + ) -> Union[tuple, MaskedImageCompletionOutput]: r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). @@ -723,9 +728,9 @@ def forward( output = (reconstructed_pixel_values,) + outputs[1:] return ((masked_im_loss,) + output) if masked_im_loss is not None else output - return MaskedLMOutput( + return MaskedImageCompletionOutput( loss=masked_im_loss, - logits=reconstructed_pixel_values, + reconstruction=reconstructed_pixel_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) diff --git a/tests/models/vit/test_modeling_vit.py b/tests/models/vit/test_modeling_vit.py index be509d460e63..d7ae3c162f59 100644 --- a/tests/models/vit/test_modeling_vit.py +++ b/tests/models/vit/test_modeling_vit.py @@ -134,7 +134,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label model.eval() result = model(pixel_values) self.parent.assertEqual( - result.logits.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) + result.reconstruction.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) ) # test greyscale images @@ -145,7 +145,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size]) result = model(pixel_values) - self.parent.assertEqual(result.logits.shape, (self.batch_size, 1, self.image_size, self.image_size)) + self.parent.assertEqual(result.reconstruction.shape, (self.batch_size, 1, self.image_size, self.image_size)) def create_and_check_for_image_classification(self, config, pixel_values, labels): config.num_labels = self.type_sequence_label_size From 730bf0c0378ca93f1991e8f2f84930c127eabaa5 Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:43:44 +0000 Subject: [PATCH 096/277] to_pil - don't rescale if int and in range 0-255 (#22158) * Don't rescale if in and in range 0-255 * Raise value error if int values too large * Update tests/test_image_transforms.py * Update tests/test_image_transforms.py --- src/transformers/image_transforms.py | 16 ++++++++++++---- tests/test_image_transforms.py | 23 ++++++++++++++++++++++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/transformers/image_transforms.py b/src/transformers/image_transforms.py index 16016d97042c..8f3fac73dd5f 100644 --- a/src/transformers/image_transforms.py +++ b/src/transformers/image_transforms.py @@ -156,12 +156,20 @@ def to_pil_image( # If there is a single channel, we squeeze it, as otherwise PIL can't handle it. image = np.squeeze(image, axis=-1) if image.shape[-1] == 1 else image - # PIL.Image can only store uint8 values, so we rescale the image to be between 0 and 255 if needed. + # PIL.Image can only store uint8 values so we rescale the image to be between 0 and 255 if needed. if do_rescale is None: - if np.all(0 <= image) and np.all(image <= 1): - do_rescale = True - elif np.allclose(image, image.astype(int)): + if image.dtype == np.uint8: do_rescale = False + elif np.allclose(image, image.astype(int)): + if np.all(0 <= image) and np.all(image <= 255): + do_rescale = False + else: + raise ValueError( + "The image to be converted to a PIL image contains values outside the range [0, 255], " + f"got [{image.min()}, {image.max()}] which cannot be converted to uint8." + ) + elif np.all(0 <= image) and np.all(image <= 1): + do_rescale = True else: raise ValueError( "The image to be converted to a PIL image contains values outside the range [0, 1], " diff --git a/tests/test_image_transforms.py b/tests/test_image_transforms.py index 0efefc7c8fbb..79580e0876e4 100644 --- a/tests/test_image_transforms.py +++ b/tests/test_image_transforms.py @@ -101,6 +101,27 @@ def test_to_pil_image_from_float(self, name, image_shape, dtype): with self.assertRaises(ValueError): to_pil_image(image) + @require_vision + def test_to_pil_image_from_mask(self): + # Make sure binary mask remains a binary mask + image = np.random.randint(0, 2, (3, 4, 5)).astype(np.uint8) + pil_image = to_pil_image(image) + self.assertIsInstance(pil_image, PIL.Image.Image) + self.assertEqual(pil_image.size, (5, 4)) + + np_img = np.asarray(pil_image) + self.assertTrue(np_img.min() == 0) + self.assertTrue(np_img.max() == 1) + + image = np.random.randint(0, 2, (3, 4, 5)).astype(np.float32) + pil_image = to_pil_image(image) + self.assertIsInstance(pil_image, PIL.Image.Image) + self.assertEqual(pil_image.size, (5, 4)) + + np_img = np.asarray(pil_image) + self.assertTrue(np_img.min() == 0) + self.assertTrue(np_img.max() == 1) + @require_tf def test_to_pil_image_from_tensorflow(self): # channels_first @@ -222,7 +243,7 @@ def test_resize(self): self.assertIsInstance(resized_image, np.ndarray) self.assertEqual(resized_image.shape, (30, 40, 3)) - # Check PIL.Image.Image is return if return_numpy=False + # Check PIL.Image.Image is returned if return_numpy=False resized_image = resize(image, (30, 40), return_numpy=False) self.assertIsInstance(resized_image, PIL.Image.Image) # PIL size is in (width, height) order From 9515690e76b3924539a07809d3db8ee4ce297e56 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 14 Mar 2023 10:22:03 -0700 Subject: [PATCH 097/277] [trainer] add `--optim adamw_torch_fused` for pt-2.0+ (#22144) * [trainer] add --optim adamw_torch_fused * change optim default * deal with non-torch * revert default change; prep; add fp16/amp assert * typo * typo --- src/transformers/trainer.py | 4 +++- src/transformers/training_args.py | 23 +++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index a5aa54fe65b2..41bb80dbc7c1 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -1122,11 +1122,13 @@ def get_optimizer_cls_and_kwargs(args: TrainingArguments) -> Tuple[Any, Any]: optimizer_cls = AdamW optimizer_kwargs.update(adam_kwargs) - elif args.optim == OptimizerNames.ADAMW_TORCH: + elif args.optim in [OptimizerNames.ADAMW_TORCH, OptimizerNames.ADAMW_TORCH_FUSED]: from torch.optim import AdamW optimizer_cls = AdamW optimizer_kwargs.update(adam_kwargs) + if args.optim == OptimizerNames.ADAMW_TORCH_FUSED: + optimizer_kwargs.update({"fused": True}) elif args.optim == OptimizerNames.ADAMW_TORCH_XLA: try: from torch_xla.amp.syncfree import AdamW diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index acce46069dc5..ce65706c0c5f 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -121,6 +121,7 @@ class OptimizerNames(ExplicitEnum): ADAMW_HF = "adamw_hf" ADAMW_TORCH = "adamw_torch" + ADAMW_TORCH_FUSED = "adamw_torch_fused" ADAMW_TORCH_XLA = "adamw_torch_xla" ADAMW_APEX_FUSED = "adamw_apex_fused" ADAFACTOR = "adafactor" @@ -457,7 +458,8 @@ class TrainingArguments: The options should be separated by whitespaces. optim (`str` or [`training_args.OptimizerNames`], *optional*, defaults to `"adamw_hf"`): - The optimizer to use: adamw_hf, adamw_torch, adamw_apex_fused, adamw_anyprecision or adafactor. + The optimizer to use: adamw_hf, adamw_torch, adamw_torch_fused, adamw_apex_fused, adamw_anyprecision or + adafactor. optim_args (`str`, *optional*): Optional arguments that are supplied to AnyPrecisionAdamW. group_by_length (`bool`, *optional*, defaults to `False`): @@ -940,8 +942,15 @@ class TrainingArguments: label_smoothing_factor: float = field( default=0.0, metadata={"help": "The label smoothing epsilon to apply (zero means no label smoothing)."} ) + + default_optim = "adamw_hf" + # XXX: enable when pytorch==2.0.1 comes out - we want to give it time to get all the bugs sorted out + # if is_torch_available() and version.parse(version.parse(torch.__version__).base_version) >= version.parse("2.1.0"): + # default_optim = "adamw_torch_fused" + # and update the doc above to: + # optim (`str` or [`training_args.OptimizerNames`], *optional*, defaults to `"adamw_torch_fused"` (for torch<2.1.0 `"adamw_hf"`): optim: Union[OptimizerNames, str] = field( - default="adamw_hf", + default=default_optim, metadata={"help": "The optimizer to use."}, ) optim_args: Optional[str] = field(default=None, metadata={"help": "Optional arguments to supply to optimizer."}) @@ -1205,6 +1214,12 @@ def __post_init__(self): FutureWarning, ) self.optim = OptimizerNames.ADAFACTOR + if self.optim == OptimizerNames.ADAMW_TORCH_FUSED and is_torch_available(): + if version.parse(version.parse(torch.__version__).base_version) < version.parse("2.0.0"): + raise ValueError("--optim adamw_torch_fused requires PyTorch 2.0 or higher") + # there is a bug in fp16/AMP in pt-2.0.0 + if version.parse(version.parse(torch.__version__).base_version) == version.parse("2.0.0") and self.fp16: + raise ValueError("--optim adamw_torch_fused with --fp16 requires PyTorch>2.0") if ( self.framework == "pt" @@ -2275,8 +2290,8 @@ def set_optimizer( Args: name (`str` or [`training_args.OptimizerNames`], *optional*, defaults to `"adamw_hf"`): - The optimizer to use: `"adamw_hf"`, `"adamw_torch"`, `"adamw_apex_fused"`, `"adamw_anyprecision"` or - `"adafactor"`. + The optimizer to use: `"adamw_hf"`, `"adamw_torch"`, `"adamw_torch_fused"`, `"adamw_apex_fused"`, + `"adamw_anyprecision"` or `"adafactor"`. learning_rate (`float`, *optional*, defaults to 5e-5): The initial learning rate. weight_decay (`float`, *optional*, defaults to 0): From a115f40bf1cac111d7e7d4e7b78a65b0f371f028 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 14 Mar 2023 13:45:46 -0400 Subject: [PATCH 098/277] Revert "Enforce same behavior as PyTorch 2.0 for older versions" (#22163) Revert "Enforce same behavior as PyTorch 2.0 for older versions (#22136)" This reverts commit 1c801d65eb42a71ea52db797af760bd96c8b113f. --- src/transformers/trainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 41bb80dbc7c1..2afa1d679b0f 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -1813,7 +1813,7 @@ def _inner_training_loop( # _total_loss_scalar is updated everytime .item() has to be called on tr_loss and stores the sum of all losses self._total_loss_scalar = 0.0 self._globalstep_last_logged = self.state.global_step - model.zero_grad(set_to_none=True) + model.zero_grad() self.control = self.callback_handler.on_train_begin(args, self.state, self.control) @@ -1971,7 +1971,7 @@ def _inner_training_loop( if optimizer_was_run and not self.deepspeed: self.lr_scheduler.step() - model.zero_grad(set_to_none=True) + model.zero_grad() self.state.global_step += 1 self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch self.control = self.callback_handler.on_step_end(args, self.state, self.control) From 79892754312bc8a33a2a14d29a4420a81a10d5b5 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger Date: Tue, 14 Mar 2023 13:49:10 -0400 Subject: [PATCH 099/277] v4.28.0.dev0 --- README.md | 28 +++++++++---------- README_es.md | 28 +++++++++---------- README_hd.md | 28 +++++++++---------- README_ja.md | 28 +++++++++---------- README_ko.md | 28 +++++++++---------- README_zh-hans.md | 28 +++++++++---------- README_zh-hant.md | 28 +++++++++---------- examples/flax/question-answering/run_qa.py | 2 +- .../flax/text-classification/run_flax_glue.py | 2 +- .../flax/token-classification/run_flax_ner.py | 2 +- .../run_audio_classification.py | 2 +- .../contrastive-image-text/run_clip.py | 2 +- .../run_image_classification.py | 2 +- .../run_image_classification_no_trainer.py | 2 +- examples/pytorch/image-pretraining/run_mae.py | 2 +- examples/pytorch/image-pretraining/run_mim.py | 2 +- examples/pytorch/language-modeling/run_clm.py | 2 +- .../language-modeling/run_clm_no_trainer.py | 2 +- examples/pytorch/language-modeling/run_mlm.py | 2 +- .../language-modeling/run_mlm_no_trainer.py | 2 +- examples/pytorch/language-modeling/run_plm.py | 2 +- examples/pytorch/multiple-choice/run_swag.py | 2 +- .../multiple-choice/run_swag_no_trainer.py | 2 +- examples/pytorch/question-answering/run_qa.py | 2 +- .../question-answering/run_qa_beam_search.py | 2 +- .../run_qa_beam_search_no_trainer.py | 2 +- .../question-answering/run_qa_no_trainer.py | 2 +- .../question-answering/run_seq2seq_qa.py | 2 +- .../run_semantic_segmentation.py | 2 +- .../run_semantic_segmentation_no_trainer.py | 2 +- .../run_speech_recognition_ctc.py | 2 +- .../run_speech_recognition_seq2seq.py | 2 +- .../summarization/run_summarization.py | 2 +- .../run_summarization_no_trainer.py | 2 +- .../pytorch/text-classification/run_glue.py | 2 +- .../run_glue_no_trainer.py | 2 +- .../pytorch/text-classification/run_xnli.py | 2 +- .../pytorch/token-classification/run_ner.py | 2 +- .../run_ner_no_trainer.py | 2 +- .../pytorch/translation/run_translation.py | 2 +- .../translation/run_translation_no_trainer.py | 2 +- .../contrastive-image-text/run_clip.py | 2 +- .../run_image_classification.py | 2 +- .../tensorflow/multiple-choice/run_swag.py | 2 +- .../tensorflow/question-answering/run_qa.py | 2 +- .../summarization/run_summarization.py | 2 +- .../text-classification/run_glue.py | 2 +- .../tensorflow/translation/run_translation.py | 2 +- setup.py | 2 +- src/transformers/__init__.py | 2 +- 50 files changed, 141 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index 17df3295ca9a..ff6da10fdf3f 100644 --- a/README.md +++ b/README.md @@ -276,7 +276,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 🤗 Transformers currently provides the following architectures (see [here](https://huggingface.co/docs/transformers/model_summary) for a high-level summary of each them): 1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. -1. **[ALIGN](https://huggingface.co/docs/transformers/main/model_doc/align)** (from Google Research) released with the paper [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. +1. **[ALIGN](https://huggingface.co/docs/transformers/model_doc/align)** (from Google Research) released with the paper [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. 1. **[AltCLIP](https://huggingface.co/docs/transformers/model_doc/altclip)** (from BAAI) released with the paper [AltCLIP: Altering the Language Encoder in CLIP for Extended Language Capabilities](https://arxiv.org/abs/2211.06679) by Chen, Zhongzhi and Liu, Guang and Zhang, Bo-Wen and Ye, Fulong and Yang, Qinghong and Wu, Ledell. 1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://arxiv.org/abs/2104.01778) by Yuan Gong, Yu-An Chung, James Glass. 1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/abs/1910.13461) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer. @@ -293,22 +293,22 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BLIP](https://huggingface.co/docs/transformers/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. -1. **[BLIP-2](https://huggingface.co/docs/transformers/main/model_doc/blip-2)** (from Salesforce) released with the paper [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) by Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. +1. **[BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2)** (from Salesforce) released with the paper [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) by Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. -1. **[BridgeTower](https://huggingface.co/docs/transformers/main/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. +1. **[BridgeTower](https://huggingface.co/docs/transformers/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. 1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 1. **[CANINE](https://huggingface.co/docs/transformers/model_doc/canine)** (from Google Research) released with the paper [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) by Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting. 1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://arxiv.org/abs/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou. -1. **[CLAP](https://huggingface.co/docs/transformers/main/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. 1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. 1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://arxiv.org/abs/2112.10003) by Timo Lüddecke and Alexander Ecker. 1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://arxiv.org/abs/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong. 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. -1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. @@ -318,7 +318,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (from Berkeley/Facebook/Google) released with the paper [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://arxiv.org/abs/2106.01345) by Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch. 1. **[Deformable DETR](https://huggingface.co/docs/transformers/model_doc/deformable_detr)** (from SenseTime Research) released with the paper [Deformable DETR: Deformable Transformers for End-to-End Object Detection](https://arxiv.org/abs/2010.04159) by Xizhou Zhu, Weijie Su, Lewei Lu, Bin Li, Xiaogang Wang, Jifeng Dai. 1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (from Facebook) released with the paper [Training data-efficient image transformers & distillation through attention](https://arxiv.org/abs/2012.12877) by Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou. -1. **[DETA](https://huggingface.co/docs/transformers/main/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. +1. **[DETA](https://huggingface.co/docs/transformers/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. 1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (from Facebook) released with the paper [End-to-End Object Detection with Transformers](https://arxiv.org/abs/2005.12872) by Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko. 1. **[DialoGPT](https://huggingface.co/docs/transformers/model_doc/dialogpt)** (from Microsoft Research) released with the paper [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan. 1. **[DiNAT](https://huggingface.co/docs/transformers/model_doc/dinat)** (from SHI Labs) released with the paper [Dilated Neighborhood Attention Transformer](https://arxiv.org/abs/2209.15001) by Ali Hassani and Humphrey Shi. @@ -328,11 +328,11 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[DPR](https://huggingface.co/docs/transformers/model_doc/dpr)** (from Facebook) released with the paper [Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) by Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih. 1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (from Intel Labs) released with the paper [Vision Transformers for Dense Prediction](https://arxiv.org/abs/2103.13413) by René Ranftl, Alexey Bochkovskiy, Vladlen Koltun. 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. -1. **[EfficientNet](https://huggingface.co/docs/transformers/main/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (from Google Research) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (from Baidu) released with the paper [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu. -1. **[ErnieM](https://huggingface.co/docs/transformers/main/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. +1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. 1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2 and ESMFold** were released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives. 1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei 1. **[FLAN-UL2](https://huggingface.co/docs/transformers/model_doc/flan-ul2)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei @@ -349,13 +349,13 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. 1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released in the repository [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki. 1. **[GPT-Sw3](https://huggingface.co/docs/transformers/model_doc/gpt-sw3)** (from AI-Sweden) released with the paper [Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish](http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf) by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman, Fredrik Carlsson, Magnus Sahlgren. -1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/main/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). +1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). 1. **[Graphormer](https://huggingface.co/docs/transformers/model_doc/graphormer)** (from Microsoft) released with the paper [Do Transformers Really Perform Bad for Graph Representation?](https://arxiv.org/abs/2106.05234) by Chengxuan Ying, Tianle Cai, Shengjie Luo, Shuxin Zheng, Guolin Ke, Di He, Yanming Shen, Tie-Yan Liu. 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://arxiv.org/abs/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang. 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://arxiv.org/abs/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed. 1. **[I-BERT](https://huggingface.co/docs/transformers/model_doc/ibert)** (from Berkeley) released with the paper [I-BERT: Integer-only BERT Quantization](https://arxiv.org/abs/2101.01321) by Sehoon Kim, Amir Gholami, Zhewei Yao, Michael W. Mahoney, Kurt Keutzer. 1. **[ImageGPT](https://huggingface.co/docs/transformers/model_doc/imagegpt)** (from OpenAI) released with the paper [Generative Pretraining from Pixels](https://openai.com/blog/image-gpt/) by Mark Chen, Alec Radford, Rewon Child, Jeffrey Wu, Heewoo Jun, David Luan, Ilya Sutskever. -1. **[Informer](https://huggingface.co/docs/transformers/main/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. +1. **[Informer](https://huggingface.co/docs/transformers/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. 1. **[Jukebox](https://huggingface.co/docs/transformers/model_doc/jukebox)** (from OpenAI) released with the paper [Jukebox: A Generative Model for Music](https://arxiv.org/pdf/2005.00341.pdf) by Prafulla Dhariwal, Heewoo Jun, Christine Payne, Jong Wook Kim, Alec Radford, Ilya Sutskever. 1. **[LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou. 1. **[LayoutLMv2](https://huggingface.co/docs/transformers/model_doc/layoutlmv2)** (from Microsoft Research Asia) released with the paper [LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding](https://arxiv.org/abs/2012.14740) by Yang Xu, Yiheng Xu, Tengchao Lv, Lei Cui, Furu Wei, Guoxin Wang, Yijuan Lu, Dinei Florencio, Cha Zhang, Wanxiang Che, Min Zhang, Lidong Zhou. @@ -378,7 +378,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. -1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. @@ -415,7 +415,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://arxiv.org/abs/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo. 1. **[SEW](https://huggingface.co/docs/transformers/model_doc/sew)** (from ASAPP) released with the paper [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi. 1. **[SEW-D](https://huggingface.co/docs/transformers/model_doc/sew_d)** (from ASAPP) released with the paper [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi. -1. **[SpeechT5](https://huggingface.co/docs/transformers/main/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. 1. **[SpeechToTextTransformer](https://huggingface.co/docs/transformers/model_doc/speech_to_text)** (from Facebook), released together with the paper [fairseq S2T: Fast Speech-to-Text Modeling with fairseq](https://arxiv.org/abs/2010.05171) by Changhan Wang, Yun Tang, Xutai Ma, Anne Wu, Dmytro Okhonko, Juan Pino. 1. **[SpeechToTextTransformer2](https://huggingface.co/docs/transformers/model_doc/speech_to_text_2)** (from Facebook), released together with the paper [Large-Scale Self- and Semi-Supervised Learning for Speech Translation](https://arxiv.org/abs/2104.06678) by Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli, Alexis Conneau. 1. **[Splinter](https://huggingface.co/docs/transformers/model_doc/splinter)** (from Tel Aviv University), released together with the paper [Few-Shot Question Answering by Pretraining Span Selection](https://arxiv.org/abs/2101.00438) by Ori Ram, Yuval Kirstain, Jonathan Berant, Amir Globerson, Omer Levy. @@ -434,7 +434,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[Trajectory Transformer](https://huggingface.co/docs/transformers/model_doc/trajectory_transformers)** (from the University of California at Berkeley) released with the paper [Offline Reinforcement Learning as One Big Sequence Modeling Problem](https://arxiv.org/abs/2106.02039) by Michael Janner, Qiyang Li, Sergey Levine 1. **[Transformer-XL](https://huggingface.co/docs/transformers/model_doc/transfo-xl)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. 1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft), released together with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. -1. **[TVLT](https://huggingface.co/docs/transformers/main/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. +1. **[TVLT](https://huggingface.co/docs/transformers/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. 1. **[UL2](https://huggingface.co/docs/transformers/model_doc/ul2)** (from Google Research) released with the paper [Unifying Language Learning Paradigms](https://arxiv.org/abs/2205.05131v1) by Yi Tay, Mostafa Dehghani, Vinh Q. Tran, Xavier Garcia, Dara Bahri, Tal Schuster, Huaixiu Steven Zheng, Neil Houlsby, Donald Metzler 1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (from Microsoft Research) released with the paper [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang. 1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (from Microsoft Research) released with the paper [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://arxiv.org/abs/2110.05752) by Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu. @@ -453,7 +453,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (from OpenAI) released with the paper [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) by Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever. 1. **[X-CLIP](https://huggingface.co/docs/transformers/model_doc/xclip)** (from Microsoft Research) released with the paper [Expanding Language-Image Pretrained Models for General Video Recognition](https://arxiv.org/abs/2208.02816) by Bolin Ni, Houwen Peng, Minghao Chen, Songyang Zhang, Gaofeng Meng, Jianlong Fu, Shiming Xiang, Haibin Ling. -1. **[X-MOD](https://huggingface.co/docs/transformers/main/model_doc/xmod)** (from Meta AI) released with the paper [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) by Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. +1. **[X-MOD](https://huggingface.co/docs/transformers/model_doc/xmod)** (from Meta AI) released with the paper [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) by Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. 1. **[XGLM](https://huggingface.co/docs/transformers/model_doc/xglm)** (From Facebook AI) released with the paper [Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668) by Xi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlm-prophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. diff --git a/README_es.md b/README_es.md index 2ae2a604a5de..57b6272d4d17 100644 --- a/README_es.md +++ b/README_es.md @@ -264,7 +264,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 🤗 Transformers actualmente proporciona las siguientes arquitecturas (ver [aquí](https://huggingface.co/docs/transformers/model_summary) para un resumen de alto nivel de cada uno de ellas.): 1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. -1. **[ALIGN](https://huggingface.co/docs/transformers/main/model_doc/align)** (from Google Research) released with the paper [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. +1. **[ALIGN](https://huggingface.co/docs/transformers/model_doc/align)** (from Google Research) released with the paper [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. 1. **[AltCLIP](https://huggingface.co/docs/transformers/model_doc/altclip)** (from BAAI) released with the paper [AltCLIP: Altering the Language Encoder in CLIP for Extended Language Capabilities](https://arxiv.org/abs/2211.06679) by Chen, Zhongzhi and Liu, Guang and Zhang, Bo-Wen and Ye, Fulong and Yang, Qinghong and Wu, Ledell. 1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://arxiv.org/abs/2104.01778) by Yuan Gong, Yu-An Chung, James Glass. 1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/abs/1910.13461) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer. @@ -281,22 +281,22 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BLIP](https://huggingface.co/docs/transformers/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. -1. **[BLIP-2](https://huggingface.co/docs/transformers/main/model_doc/blip-2)** (from Salesforce) released with the paper [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) by Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. +1. **[BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2)** (from Salesforce) released with the paper [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) by Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. -1. **[BridgeTower](https://huggingface.co/docs/transformers/main/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. +1. **[BridgeTower](https://huggingface.co/docs/transformers/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. 1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 1. **[CANINE](https://huggingface.co/docs/transformers/model_doc/canine)** (from Google Research) released with the paper [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) by Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting. 1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://arxiv.org/abs/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou. -1. **[CLAP](https://huggingface.co/docs/transformers/main/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. 1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. 1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://arxiv.org/abs/2112.10003) by Timo Lüddecke and Alexander Ecker. 1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://arxiv.org/abs/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong. 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. -1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. @@ -306,7 +306,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (from Berkeley/Facebook/Google) released with the paper [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://arxiv.org/abs/2106.01345) by Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch. 1. **[Deformable DETR](https://huggingface.co/docs/transformers/model_doc/deformable_detr)** (from SenseTime Research) released with the paper [Deformable DETR: Deformable Transformers for End-to-End Object Detection](https://arxiv.org/abs/2010.04159) by Xizhou Zhu, Weijie Su, Lewei Lu, Bin Li, Xiaogang Wang, Jifeng Dai. 1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (from Facebook) released with the paper [Training data-efficient image transformers & distillation through attention](https://arxiv.org/abs/2012.12877) by Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou. -1. **[DETA](https://huggingface.co/docs/transformers/main/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. +1. **[DETA](https://huggingface.co/docs/transformers/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. 1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (from Facebook) released with the paper [End-to-End Object Detection with Transformers](https://arxiv.org/abs/2005.12872) by Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko. 1. **[DialoGPT](https://huggingface.co/docs/transformers/model_doc/dialogpt)** (from Microsoft Research) released with the paper [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan. 1. **[DiNAT](https://huggingface.co/docs/transformers/model_doc/dinat)** (from SHI Labs) released with the paper [Dilated Neighborhood Attention Transformer](https://arxiv.org/abs/2209.15001) by Ali Hassani and Humphrey Shi. @@ -316,11 +316,11 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[DPR](https://huggingface.co/docs/transformers/model_doc/dpr)** (from Facebook) released with the paper [Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) by Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih. 1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (from Intel Labs) released with the paper [Vision Transformers for Dense Prediction](https://arxiv.org/abs/2103.13413) by René Ranftl, Alexey Bochkovskiy, Vladlen Koltun. 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. -1. **[EfficientNet](https://huggingface.co/docs/transformers/main/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (from Google Research) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (from Baidu) released with the paper [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu. -1. **[ErnieM](https://huggingface.co/docs/transformers/main/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. +1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. 1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2** was released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives. 1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei 1. **[FLAN-UL2](https://huggingface.co/docs/transformers/model_doc/flan-ul2)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei @@ -337,13 +337,13 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. 1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released in the repository [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki. 1. **[GPT-Sw3](https://huggingface.co/docs/transformers/model_doc/gpt-sw3)** (from AI-Sweden) released with the paper [Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish](http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf) by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman, Fredrik Carlsson, Magnus Sahlgren. -1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/main/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). +1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). 1. **[Graphormer](https://huggingface.co/docs/transformers/model_doc/graphormer)** (from Microsoft) released with the paper [Do Transformers Really Perform Bad for Graph Representation?](https://arxiv.org/abs/2106.05234) by Chengxuan Ying, Tianle Cai, Shengjie Luo, Shuxin Zheng, Guolin Ke, Di He, Yanming Shen, Tie-Yan Liu. 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://arxiv.org/abs/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang. 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://arxiv.org/abs/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed. 1. **[I-BERT](https://huggingface.co/docs/transformers/model_doc/ibert)** (from Berkeley) released with the paper [I-BERT: Integer-only BERT Quantization](https://arxiv.org/abs/2101.01321) by Sehoon Kim, Amir Gholami, Zhewei Yao, Michael W. Mahoney, Kurt Keutzer. 1. **[ImageGPT](https://huggingface.co/docs/transformers/model_doc/imagegpt)** (from OpenAI) released with the paper [Generative Pretraining from Pixels](https://openai.com/blog/image-gpt/) by Mark Chen, Alec Radford, Rewon Child, Jeffrey Wu, Heewoo Jun, David Luan, Ilya Sutskever. -1. **[Informer](https://huggingface.co/docs/transformers/main/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. +1. **[Informer](https://huggingface.co/docs/transformers/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. 1. **[Jukebox](https://huggingface.co/docs/transformers/model_doc/jukebox)** (from OpenAI) released with the paper [Jukebox: A Generative Model for Music](https://arxiv.org/pdf/2005.00341.pdf) by Prafulla Dhariwal, Heewoo Jun, Christine Payne, Jong Wook Kim, Alec Radford, Ilya Sutskever. 1. **[LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou. 1. **[LayoutLMv2](https://huggingface.co/docs/transformers/model_doc/layoutlmv2)** (from Microsoft Research Asia) released with the paper [LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding](https://arxiv.org/abs/2012.14740) by Yang Xu, Yiheng Xu, Tengchao Lv, Lei Cui, Furu Wei, Guoxin Wang, Yijuan Lu, Dinei Florencio, Cha Zhang, Wanxiang Che, Min Zhang, Lidong Zhou. @@ -366,7 +366,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. -1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. @@ -403,7 +403,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://arxiv.org/abs/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo. 1. **[SEW](https://huggingface.co/docs/transformers/model_doc/sew)** (from ASAPP) released with the paper [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi. 1. **[SEW-D](https://huggingface.co/docs/transformers/model_doc/sew_d)** (from ASAPP) released with the paper [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi. -1. **[SpeechT5](https://huggingface.co/docs/transformers/main/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. 1. **[SpeechToTextTransformer](https://huggingface.co/docs/transformers/model_doc/speech_to_text)** (from Facebook), released together with the paper [fairseq S2T: Fast Speech-to-Text Modeling with fairseq](https://arxiv.org/abs/2010.05171) by Changhan Wang, Yun Tang, Xutai Ma, Anne Wu, Dmytro Okhonko, Juan Pino. 1. **[SpeechToTextTransformer2](https://huggingface.co/docs/transformers/model_doc/speech_to_text_2)** (from Facebook), released together with the paper [Large-Scale Self- and Semi-Supervised Learning for Speech Translation](https://arxiv.org/abs/2104.06678) by Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli, Alexis Conneau. 1. **[Splinter](https://huggingface.co/docs/transformers/model_doc/splinter)** (from Tel Aviv University), released together with the paper [Few-Shot Question Answering by Pretraining Span Selection](https://arxiv.org/abs/2101.00438) by Ori Ram, Yuval Kirstain, Jonathan Berant, Amir Globerson, Omer Levy. @@ -422,7 +422,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[Trajectory Transformer](https://huggingface.co/docs/transformers/model_doc/trajectory_transformers)** (from the University of California at Berkeley) released with the paper [Offline Reinforcement Learning as One Big Sequence Modeling Problem](https://arxiv.org/abs/2106.02039) by Michael Janner, Qiyang Li, Sergey Levine 1. **[Transformer-XL](https://huggingface.co/docs/transformers/model_doc/transfo-xl)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. 1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft), released together with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. -1. **[TVLT](https://huggingface.co/docs/transformers/main/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. +1. **[TVLT](https://huggingface.co/docs/transformers/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. 1. **[UL2](https://huggingface.co/docs/transformers/model_doc/ul2)** (from Google Research) released with the paper [Unifying Language Learning Paradigms](https://arxiv.org/abs/2205.05131v1) by Yi Tay, Mostafa Dehghani, Vinh Q. Tran, Xavier Garcia, Dara Bahri, Tal Schuster, Huaixiu Steven Zheng, Neil Houlsby, Donald Metzler 1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (from Microsoft Research) released with the paper [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang. 1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (from Microsoft Research) released with the paper [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://arxiv.org/abs/2110.05752) by Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu. @@ -441,7 +441,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (from OpenAI) released with the paper [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) by Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever. 1. **[X-CLIP](https://huggingface.co/docs/transformers/model_doc/xclip)** (from Microsoft Research) released with the paper [Expanding Language-Image Pretrained Models for General Video Recognition](https://arxiv.org/abs/2208.02816) by Bolin Ni, Houwen Peng, Minghao Chen, Songyang Zhang, Gaofeng Meng, Jianlong Fu, Shiming Xiang, Haibin Ling. -1. **[X-MOD](https://huggingface.co/docs/transformers/main/model_doc/xmod)** (from Meta AI) released with the paper [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) by Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. +1. **[X-MOD](https://huggingface.co/docs/transformers/model_doc/xmod)** (from Meta AI) released with the paper [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) by Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. 1. **[XGLM](https://huggingface.co/docs/transformers/model_doc/xglm)** (From Facebook AI) released with the paper [Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668) by Xi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlm-prophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. diff --git a/README_hd.md b/README_hd.md index 9dffd0be1b77..2355369ad8d3 100644 --- a/README_hd.md +++ b/README_hd.md @@ -236,7 +236,7 @@ conda install -c huggingface transformers 🤗 ट्रांसफॉर्मर वर्तमान में निम्नलिखित आर्किटेक्चर का समर्थन करते हैं (मॉडल के अवलोकन के लिए [यहां] देखें (https://huggingface.co/docs/transformers/model_summary))： 1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (Google Research and the Toyota Technological Institute at Chicago) साथ थीसिस [ALBERT: A Lite BERT for Self-supervised भाषा प्रतिनिधित्व सीखना](https://arxiv.org/abs/1909.11942), झेंझोंग लैन, मिंगदा चेन, सेबेस्टियन गुडमैन, केविन गिम्पेल, पीयूष शर्मा, राडू सोरिकट -1. **[ALIGN](https://huggingface.co/docs/transformers/main/model_doc/align)** (Google Research से) Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. द्वाराअनुसंधान पत्र [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) के साथ जारी किया गया +1. **[ALIGN](https://huggingface.co/docs/transformers/model_doc/align)** (Google Research से) Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. द्वाराअनुसंधान पत्र [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) के साथ जारी किया गया 1. **[AltCLIP](https://huggingface.co/docs/transformers/model_doc/altclip)** (from BAAI) released with the paper [AltCLIP: Altering the Language Encoder in CLIP for Extended Language Capabilities](https://arxiv.org/abs/2211.06679) by Chen, Zhongzhi and Liu, Guang and Zhang, Bo-Wen and Ye, Fulong and Yang, Qinghong and Wu, Ledell. 1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://arxiv.org/abs/2104.01778) by Yuan Gong, Yu-An Chung, James Glass. 1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (फेसबुक) साथ थीसिस [बार्ट: प्राकृतिक भाषा निर्माण, अनुवाद के लिए अनुक्रम-से-अनुक्रम पूर्व प्रशिक्षण , और समझ] (https://arxiv.org/pdf/1910.13461.pdf) पर निर्भर माइक लुईस, यिनहान लियू, नमन गोयल, मार्जन ग़ज़विनिनेजाद, अब्देलरहमान मोहम्मद, ओमर लेवी, वेस स्टोयानोव और ल्यूक ज़ेटलमॉयर @@ -253,22 +253,22 @@ conda install -c huggingface transformers 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (फेसबुक से) साथ में कागज [एक ओपन-डोमेन चैटबॉट बनाने की विधि](https://arxiv.org /abs/2004.13637) स्टीफन रोलर, एमिली दीनन, नमन गोयल, दा जू, मैरी विलियमसन, यिनहान लियू, जिंग जू, मायल ओट, कर्ट शस्टर, एरिक एम। स्मिथ, वाई-लैन बॉरो, जेसन वेस्टन द्वारा। 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (फेसबुक से) साथ में पेपर [एक ओपन-डोमेन चैटबॉट बनाने की रेसिपी](https://arxiv .org/abs/2004.13637) स्टीफन रोलर, एमिली दीनन, नमन गोयल, दा जू, मैरी विलियमसन, यिनहान लियू, जिंग जू, मायल ओट, कर्ट शस्टर, एरिक एम स्मिथ, वाई-लैन बॉरो, जेसन वेस्टन द्वारा। 1. **[BLIP](https://huggingface.co/docs/transformers/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. -1. **[BLIP-2](https://huggingface.co/docs/transformers/main/model_doc/blip-2)** (Salesforce से) Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. द्वाराअनुसंधान पत्र [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) के साथ जारी किया गया +1. **[BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2)** (Salesforce से) Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. द्वाराअनुसंधान पत्र [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) के साथ जारी किया गया 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigSicence Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (एलेक्सा से) कागज के साथ [बीईआरटी के लिए ऑप्टिमल सबआर्किटेक्चर एक्सट्रैक्शन](https://arxiv.org/abs/ 2010.10499) एड्रियन डी विंटर और डैनियल जे पेरी द्वारा। -1. **[BridgeTower](https://huggingface.co/docs/transformers/main/model_doc/bridgetower)** (हरबिन इंस्टिट्यूट ऑफ़ टेक्नोलॉजी/माइक्रोसॉफ्ट रिसर्च एशिया/इंटेल लैब्स से) कागज के साथ [ब्रिजटॉवर: विजन-लैंग्वेज रिप्रेजेंटेशन लर्निंग में एनकोडर्स के बीच ब्रिज बनाना]() by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. +1. **[BridgeTower](https://huggingface.co/docs/transformers/model_doc/bridgetower)** (हरबिन इंस्टिट्यूट ऑफ़ टेक्नोलॉजी/माइक्रोसॉफ्ट रिसर्च एशिया/इंटेल लैब्स से) कागज के साथ [ब्रिजटॉवर: विजन-लैंग्वेज रिप्रेजेंटेशन लर्निंग में एनकोडर्स के बीच ब्रिज बनाना]() by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (Google अनुसंधान से) साथ में कागज [ByT5: पूर्व-प्रशिक्षित बाइट-टू-बाइट मॉडल के साथ एक टोकन-मुक्त भविष्य की ओर] (https://arxiv.org/abs/2105.13626) Linting Xue, Aditya Barua, Noah Constant, रामी अल-रफू, शरण नारंग, मिहिर काले, एडम रॉबर्ट्स, कॉलिन रैफेल द्वारा पोस्ट किया गया। 1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (इनरिया/फेसबुक/सोरबोन से) साथ में कागज [CamemBERT: एक टेस्टी फ्रेंच लैंग्वेज मॉडल](https:// arxiv.org/abs/1911.03894) लुई मार्टिन*, बेंजामिन मुलर*, पेड्रो जेवियर ऑर्टिज़ सुआरेज़*, योआन ड्यूपॉन्ट, लॉरेंट रोमरी, एरिक विलेमोन्टे डे ला क्लर्जरी, जैमे सेडाह और बेनोइट सगोट द्वारा। 1. **[CANINE](https://huggingface.co/docs/transformers/model_doc/canine)** (Google रिसर्च से) साथ में दिया गया पेपर [कैनाइन: प्री-ट्रेनिंग ए एफिशिएंट टोकनाइजेशन-फ्री एनकोडर फॉर लैंग्वेज रिप्रेजेंटेशन]( https://arxiv.org/abs/2103.06874) जोनाथन एच क्लार्क, डैन गैरेट, यूलिया टर्क, जॉन विएटिंग द्वारा। 1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://arxiv.org/abs/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou. -1. **[CLAP](https://huggingface.co/docs/transformers/main/model_doc/clap)** (LAION-AI से) Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. द्वाराअनुसंधान पत्र [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) के साथ जारी किया गया +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (LAION-AI से) Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. द्वाराअनुसंधान पत्र [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) के साथ जारी किया गया 1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (OpenAI से) साथ वाला पेपर [लर्निंग ट्रांसफरेबल विजुअल मॉडल फ्रॉम नेचुरल लैंग्वेज सुपरविजन](https://arxiv.org /abs/2103.00020) एलेक रैडफोर्ड, जोंग वूक किम, क्रिस हैलासी, आदित्य रमेश, गेब्रियल गोह, संध्या अग्रवाल, गिरीश शास्त्री, अमांडा एस्केल, पामेला मिश्किन, जैक क्लार्क, ग्रेचेन क्रुएगर, इल्या सुत्स्केवर द्वारा। 1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://arxiv.org/abs/2112.10003) by Timo Lüddecke and Alexander Ecker. 1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (सेल्सफोर्स से) साथ में पेपर [प्रोग्राम सिंथेसिस के लिए एक संवादात्मक प्रतिमान](https://arxiv.org/abs/2203.13474) एरिक निजकैंप, बो पैंग, हिरोआकी हयाशी, लिफू तू, हुआन वांग, यिंगबो झोउ, सिल्वियो सावरेस, कैमिंग जिओंग रिलीज। 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (माइक्रोसॉफ्ट रिसर्च एशिया से) कागज के साथ [फास्ट ट्रेनिंग कन्वर्जेंस के लिए सशर्त डीईटीआर](https://arxiv. org/abs/2108.06152) डेपू मेंग, ज़ियाओकांग चेन, ज़ेजिया फैन, गैंग ज़ेंग, होउकियांग ली, युहुई युआन, लेई सन, जिंगडोंग वांग द्वारा। 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (YituTech से) साथ में कागज [ConvBERT: स्पैन-आधारित डायनेमिक कनवल्शन के साथ BERT में सुधार](https://arxiv .org/abs/2008.02496) जिहांग जियांग, वीहाओ यू, डाकान झोउ, युनपेंग चेन, जियाशी फेंग, शुइचेंग यान द्वारा। 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (Facebook AI से) साथ वाला पेपर [A ConvNet for the 2020s](https://arxiv.org/abs /2201.03545) ज़ुआंग लियू, हेंज़ी माओ, चाओ-युआन वू, क्रिस्टोफ़ फीचटेनहोफ़र, ट्रेवर डेरेल, सैनिंग ज़ी द्वारा। -1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (सिंघुआ यूनिवर्सिटी से) साथ में पेपर [सीपीएम: ए लार्ज-स्केल जेनेरेटिव चाइनीज प्री-ट्रेंड लैंग्वेज मॉडल](https : //arxiv.org/abs/2012.00413) झेंग्यान झांग, जू हान, हाओ झोउ, पेई के, युक्सियन गु, डेमिंग ये, युजिया किन, युशेंग सु, हाओझे जी, जियान गुआन, फैंचाओ क्यूई, ज़ियाओझी वांग, यानान झेंग द्वारा , गुओयांग ज़ेंग, हुआनकी काओ, शेंगकी चेन, डाइक्सुआन ली, ज़ेनबो सन, ज़ियुआन लियू, मिनली हुआंग, वेंटाओ हान, जी तांग, जुआनज़ी ली, ज़ियाओयान झू, माओसोंग सन। 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (सेल्सफोर्स से) साथ में पेपर [CTRL: ए कंडिशनल ट्रांसफॉर्मर लैंग्वेज मॉडल फॉर कंट्रोलेबल जेनरेशन](https://arxiv.org/abs/1909.05858) नीतीश शिरीष केसकर*, ब्रायन मैककैन*, लव आर. वार्ष्णेय, कैमिंग जिओंग और रिचर्ड द्वारा सोचर द्वारा जारी किया गया। 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (Microsoft से) साथ में दिया गया पेपर [CvT: इंट्रोड्यूसिंग कनवॉल्यूशन टू विजन ट्रांसफॉर्मर्स](https://arxiv.org/ एब्स/2103.15808) हैपिंग वू, बिन जिओ, नोएल कोडेला, मेंगचेन लियू, जियांग दाई, लू युआन, लेई झांग द्वारा। @@ -278,7 +278,7 @@ conda install -c huggingface transformers 1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (बर्कले/फेसबुक/गूगल से) पेपर के साथ [डिसीजन ट्रांसफॉर्मर: रीनफोर्समेंट लर्निंग वाया सीक्वेंस मॉडलिंग](https : //arxiv.org/abs/2106.01345) लिली चेन, केविन लू, अरविंद राजेश्वरन, किमिन ली, आदित्य ग्रोवर, माइकल लास्किन, पीटर एबील, अरविंद श्रीनिवास, इगोर मोर्डच द्वारा पोस्ट किया गया। 1. **[Deformable DETR](https://huggingface.co/docs/transformers/model_doc/deformable_detr)** (सेंसटाइम रिसर्च से) साथ में पेपर [डिफॉर्मेबल डीईटीआर: डिफॉर्मेबल ट्रांसफॉर्मर्स फॉर एंड-टू-एंड ऑब्जेक्ट डिटेक्शन] (https://arxiv.org/abs/2010.04159) Xizhou Zhu, Weijie Su, Lewei Lu, Bin Li, Xiaogang Wang, जिफेंग दाई द्वारा पोस्ट किया गया। 1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (फेसबुक से) साथ में पेपर [ट्रेनिंग डेटा-एफिशिएंट इमेज ट्रांसफॉर्मर और डिस्टिलेशन थ्रू अटेंशन](https://arxiv .org/abs/2012.12877) ह्यूगो टौव्रोन, मैथ्यू कॉर्ड, मैथिज्स डूज़, फ़्रांसिस्को मस्सा, एलेक्ज़ेंडर सबलेरोल्स, हर्वे जेगौ द्वारा। -1. **[DETA](https://huggingface.co/docs/transformers/main/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. +1. **[DETA](https://huggingface.co/docs/transformers/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. 1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (फेसबुक से) साथ में कागज [ट्रांसफॉर्मर्स के साथ एंड-टू-एंड ऑब्जेक्ट डिटेक्शन](https://arxiv. org/abs/2005.12872) निकोलस कैरियन, फ़्रांसिस्को मस्सा, गेब्रियल सिनेव, निकोलस उसुनियर, अलेक्जेंडर किरिलोव, सर्गेई ज़ागोरुयको द्वारा। 1. **[DialoGPT](https://huggingface.co/docs/transformers/model_doc/dialogpt)** (माइक्रोसॉफ्ट रिसर्च से) कागज के साथ [DialoGPT: बड़े पैमाने पर जनरेटिव प्री-ट्रेनिंग फॉर कन्वर्सेशनल रिस्पांस जेनरेशन](https ://arxiv.org/abs/1911.00536) यिज़े झांग, सिकी सन, मिशेल गैली, येन-चुन चेन, क्रिस ब्रोकेट, जियांग गाओ, जियानफेंग गाओ, जिंगजिंग लियू, बिल डोलन द्वारा। 1. **[DiNAT](https://huggingface.co/docs/transformers/model_doc/dinat)** (from SHI Labs) released with the paper [Dilated Neighborhood Attention Transformer](https://arxiv.org/abs/2209.15001) by Ali Hassani and Humphrey Shi. @@ -288,11 +288,11 @@ conda install -c huggingface transformers 1. **[DPR](https://huggingface.co/docs/transformers/model_doc/dpr)** (फेसबुक से) साथ में पेपर [ओपन-डोमेन क्वेश्चन आंसरिंग के लिए डेंस पैसेज रिट्रीवल](https://arxiv. org/abs/2004.04906) व्लादिमीर करपुखिन, बरलास ओज़ुज़, सेवन मिन, पैट्रिक लुईस, लेडेल वू, सर्गेई एडुनोव, डैनकी चेन, और वेन-ताऊ यिह द्वारा। 1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (इंटेल लैब्स से) साथ में कागज [विज़न ट्रांसफॉर्मर्स फॉर डेंस प्रेडिक्शन](https://arxiv.org /abs/2103.13413) रेने रैनफ्टल, एलेक्सी बोचकोवस्की, व्लादलेन कोल्टन द्वारा। 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. -1. **[EfficientNet](https://huggingface.co/docs/transformers/main/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (Google रिसर्च/स्टैनफोर्ड यूनिवर्सिटी से) साथ में दिया गया पेपर [इलेक्ट्रा: जेनरेटर के बजाय भेदभाव करने वाले के रूप में टेक्स्ट एन्कोडर्स का पूर्व-प्रशिक्षण] (https://arxiv.org/abs/2003.10555) केविन क्लार्क, मिन्ह-थांग लुओंग, क्वोक वी. ले, क्रिस्टोफर डी. मैनिंग द्वारा पोस्ट किया गया। 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (Google रिसर्च से) साथ में दिया गया पेपर [सीक्वेंस जेनरेशन टास्क के लिए प्री-ट्रेंड चेकपॉइंट का इस्तेमाल करना](https:/ /arxiv.org/abs/1907.12461) साशा रोठे, शशि नारायण, अलियाक्सि सेवेरिन द्वारा। 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)**(Baidu से) साथ देने वाला पेपर [ERNIE: एन्हांस्ड रिप्रेजेंटेशन थ्रू नॉलेज इंटीग्रेशन](https://arxiv.org/abs/1904.09223) यू सन, शुओहुआन वांग, युकुन ली, शिकुन फेंग, ज़ुई चेन, हान झांग, शिन तियान, डैनक्सियांग झू, हाओ तियान, हुआ वू द्वारा पोस्ट किया गया। -1. **[ErnieM](https://huggingface.co/docs/transformers/main/model_doc/ernie_m)** (Baidu से) Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. द्वाराअनुसंधान पत्र [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) के साथ जारी किया गया +1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (Baidu से) Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. द्वाराअनुसंधान पत्र [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) के साथ जारी किया गया 1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (मेटा AI से) ट्रांसफॉर्मर प्रोटीन भाषा मॉडल हैं। **ESM-1b** पेपर के साथ जारी किया गया था [ अलेक्जेंडर राइव्स, जोशुआ मेयर, टॉम सर्कु, सिद्धार्थ गोयल, ज़ेमिंग लिन द्वारा जैविक संरचना और कार्य असुरक्षित सीखने को 250 मिलियन प्रोटीन अनुक्रमों तक स्केल करने से उभरता है] (https://www.pnas.org/content/118/15/e2016239118) जेसन लियू, डेमी गुओ, मायल ओट, सी. लॉरेंस ज़िटनिक, जेरी मा और रॉब फर्गस। **ESM-1v** को पेपर के साथ जारी किया गया था [भाषा मॉडल प्रोटीन फ़ंक्शन पर उत्परिवर्तन के प्रभावों की शून्य-शॉट भविष्यवाणी को सक्षम करते हैं] (https://doi.org/10.1101/2021.07.09.450648) जोशुआ मेयर, रोशन राव, रॉबर्ट वेरकुइल, जेसन लियू, टॉम सर्कु और अलेक्जेंडर राइव्स द्वारा। **ESM-2** को पेपर के साथ जारी किया गया था [भाषा मॉडल विकास के पैमाने पर प्रोटीन अनुक्रम सटीक संरचना भविष्यवाणी को सक्षम करते हैं](https://doi.org/10.1101/2022.07.20.500902) ज़ेमिंग लिन, हलील अकिन, रोशन राव, ब्रायन ही, झोंगकाई झू, वेंटिंग लू, ए द्वारा लान डॉस सैंटोस कोस्टा, मरियम फ़ज़ल-ज़रंडी, टॉम सर्कू, साल कैंडिडो, अलेक्जेंडर राइव्स। 1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei 1. **[FLAN-UL2](https://huggingface.co/docs/transformers/model_doc/flan-ul2)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei @@ -309,13 +309,13 @@ conda install -c huggingface transformers 1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (ओपनएआई से) साथ में पेपर [लैंग्वेज मॉडल्स अनसुपरवाइज्ड मल्टीटास्क लर्नर्स हैं](https://blog.openai.com/better-language-models/) एलेक रैडफोर्ड*, जेफरी वू*, रेवन चाइल्ड, डेविड लुआन, डारियो एमोडी* द्वारा * और इल्या सुत्सकेवर** ने पोस्ट किया। 1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (EleutherAI से) साथ वाला पेपर [kingoflolz/mesh-transformer-jax](https://github. com/kingoflolz/mesh-transformer-jax/) बेन वांग और अरन कोमात्सुजाकी द्वारा। 1. **[GPT-Sw3](https://huggingface.co/docs/transformers/model_doc/gpt-sw3)** (from AI-Sweden) released with the paper [Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish](http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf) by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman, Fredrik Carlsson, Magnus Sahlgren. -1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/main/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). +1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). 1. **[Graphormer](https://huggingface.co/docs/transformers/model_doc/graphormer)** (from Microsoft) released with the paper [Do Transformers Really Perform Bad for Graph Representation?](https://arxiv.org/abs/2106.05234) by Chengxuan Ying, Tianle Cai, Shengjie Luo, Shuxin Zheng, Guolin Ke, Di He, Yanming Shen, Tie-Yan Liu. 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (UCSD, NVIDIA से) साथ में कागज [GroupViT: टेक्स्ट सुपरविजन से सिमेंटिक सेगमेंटेशन इमर्जेस](https://arxiv .org/abs/2202.11094) जियारुई जू, शालिनी डी मेलो, सिफ़ी लियू, वोनमिन बायन, थॉमस ब्रेउएल, जान कौट्ज़, ज़ियाओलोंग वांग द्वारा। 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (फेसबुक से) साथ में पेपर [ह्यूबर्ट: सेल्फ सुपरवाइज्ड स्पीच रिप्रेजेंटेशन लर्निंग बाय मास्क्ड प्रेडिक्शन ऑफ हिडन यूनिट्स](https ://arxiv.org/abs/2106.07447) वेई-निंग सू, बेंजामिन बोल्टे, याओ-हंग ह्यूबर्ट त्साई, कुशाल लखोटिया, रुस्लान सालाखुतदीनोव, अब्देलरहमान मोहम्मद द्वारा। 1. **[I-BERT](https://huggingface.co/docs/transformers/model_doc/ibert)** (बर्कले से) साथ में कागज [I-BERT: Integer-only BERT Quantization](https:// arxiv.org/abs/2101.01321) सेहून किम, अमीर घोलमी, ज़ेवेई याओ, माइकल डब्ल्यू महोनी, कर्ट केटज़र द्वारा। 1. **[ImageGPT](https://huggingface.co/docs/transformers/model_doc/imagegpt)** (from OpenAI) released with the paper [Generative Pretraining from Pixels](https://openai.com/blog/image-gpt/) by Mark Chen, Alec Radford, Rewon Child, Jeffrey Wu, Heewoo Jun, David Luan, Ilya Sutskever. -1. **[Informer](https://huggingface.co/docs/transformers/main/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. +1. **[Informer](https://huggingface.co/docs/transformers/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. 1. **[Jukebox](https://huggingface.co/docs/transformers/model_doc/jukebox)** (from OpenAI) released with the paper [Jukebox: A Generative Model for Music](https://arxiv.org/pdf/2005.00341.pdf) by Prafulla Dhariwal, Heewoo Jun, Christine Payne, Jong Wook Kim, Alec Radford, Ilya Sutskever. 1. **[LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou. 1. **[LayoutLMv2](https://huggingface.co/docs/transformers/model_doc/layoutlmv2)** (from Microsoft Research Asia) released with the paper [LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding](https://arxiv.org/abs/2012.14740) by Yang Xu, Yiheng Xu, Tengchao Lv, Lei Cui, Furu Wei, Guoxin Wang, Yijuan Lu, Dinei Florencio, Cha Zhang, Wanxiang Che, Min Zhang, Lidong Zhou. @@ -338,7 +338,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (फेसबुक से) साथ में पेपर [एक्स्टेंसिबल बहुभाषी प्रीट्रेनिंग और फाइनट्यूनिंग के साथ बहुभाषी अनुवाद](https://arxiv युकिंग टैंग, चाउ ट्रान, जियान ली, पेंग-जेन चेन, नमन गोयल, विश्रव चौधरी, जियाताओ गु, एंजेला फैन द्वारा .org/abs/2008.00401)। 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA से) कागज के साथ [Megatron-LM: मॉडल का उपयोग करके बहु-अरब पैरामीटर भाषा मॉडल का प्रशिक्षण Parallelism](https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा। 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA से) साथ वाला पेपर [Megatron-LM: ट्रेनिंग मल्टी-बिलियन पैरामीटर लैंग्वेज मॉडल्स यूजिंग मॉडल पैरेललिज़्म] (https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा पोस्ट किया गया। -1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (Alibaba Research से) Peng Wang, Cheng Da, and Cong Yao. द्वाराअनुसंधान पत्र [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) के साथ जारी किया गया +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research से) Peng Wang, Cheng Da, and Cong Yao. द्वाराअनुसंधान पत्र [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) के साथ जारी किया गया 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (फ्रॉम Studio Ousia) साथ में पेपर [mLUKE: द पावर ऑफ एंटिटी रिप्रेजेंटेशन इन मल्टीलिंगुअल प्रीट्रेन्ड लैंग्वेज मॉडल्स](https://arxiv.org/abs/2110.08151) रयोकन री, इकुया यामाडा, और योशिमासा त्सुरोका द्वारा। 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (सीएमयू/गूगल ब्रेन से) साथ में कागज [मोबाइलबर्ट: संसाधन-सीमित उपकरणों के लिए एक कॉम्पैक्ट टास्क-अज्ञेय बीईआरटी] (https://arxiv.org/abs/2004.02984) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, और Denny Zhou द्वारा पोस्ट किया गया। 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. @@ -375,7 +375,7 @@ conda install -c huggingface transformers 1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://arxiv.org/abs/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo. 1. **[SEW](https://huggingface.co/docs/transformers/model_doc/sew)** (ASAPP से) साथ देने वाला पेपर [भाषण पहचान के लिए अनसुपरवाइज्ड प्री-ट्रेनिंग में परफॉर्मेंस-एफिशिएंसी ट्रेड-ऑफ्स](https ://arxiv.org/abs/2109.06870) फेलिक्स वू, क्वांगयुन किम, जिंग पैन, क्यू हान, किलियन क्यू. वेनबर्गर, योव आर्टज़ी द्वारा। 1. **[SEW-D](https://huggingface.co/docs/transformers/model_doc/sew_d)** (ASAPP से) साथ में पेपर [भाषण पहचान के लिए अनसुपरवाइज्ड प्री-ट्रेनिंग में परफॉर्मेंस-एफिशिएंसी ट्रेड-ऑफ्स] (https://arxiv.org/abs/2109.06870) फेलिक्स वू, क्वांगयुन किम, जिंग पैन, क्यू हान, किलियन क्यू. वेनबर्गर, योआव आर्टज़ी द्वारा पोस्ट किया गया। -1. **[SpeechT5](https://huggingface.co/docs/transformers/main/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. 1. **[SpeechToTextTransformer](https://huggingface.co/docs/transformers/model_doc/speech_to_text)** (फेसबुक से), साथ में पेपर [फेयरसेक S2T: फास्ट स्पीच-टू-टेक्स्ट मॉडलिंग विद फेयरसेक](https: //arxiv.org/abs/2010.05171) चांगहान वांग, यूं तांग, जुताई मा, ऐनी वू, दिमित्रो ओखोनको, जुआन पिनो द्वारा पोस्ट किया गया。 1. **[SpeechToTextTransformer2](https://huggingface.co/docs/transformers/model_doc/speech_to_text_2)** (फेसबुक से) साथ में पेपर [लार्ज-स्केल सेल्फ- एंड सेमी-सुपरवाइज्ड लर्निंग फॉर स्पीच ट्रांसलेशन](https://arxiv.org/abs/2104.06678) चांगहान वांग, ऐनी वू, जुआन पिनो, एलेक्सी बेवस्की, माइकल औली, एलेक्सिस द्वारा Conneau द्वारा पोस्ट किया गया। 1. **[Splinter](https://huggingface.co/docs/transformers/model_doc/splinter)** (तेल अवीव यूनिवर्सिटी से) साथ में पेपर [स्पैन सिलेक्शन को प्री-ट्रेनिंग करके कुछ-शॉट क्वेश्चन आंसरिंग](https:// arxiv.org/abs/2101.00438) ओरि राम, युवल कर्स्टन, जोनाथन बेरेंट, अमीर ग्लोबर्सन, ओमर लेवी द्वारा। @@ -394,7 +394,7 @@ conda install -c huggingface transformers 1. **[Trajectory Transformer](https://huggingface.co/docs/transformers/model_doc/trajectory_transformers)** (from the University of California at Berkeley) released with the paper [Offline Reinforcement Learning as One Big Sequence Modeling Problem](https://arxiv.org/abs/2106.02039) by Michael Janner, Qiyang Li, Sergey Levine 1. **[Transformer-XL](https://huggingface.co/docs/transformers/model_doc/transfo-xl)** (Google/CMU की ओर से) कागज के साथ [संस्करण-एक्स: एक ब्लॉग मॉडल चौकस चौक मॉडल मॉडल] (https://arxivorg/abs/1901.02860) क्वोकोक वी. ले, रुस्लैन सलाखुतदी 1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft) released with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. -1. **[TVLT](https://huggingface.co/docs/transformers/main/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. +1. **[TVLT](https://huggingface.co/docs/transformers/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. 1. **[UL2](https://huggingface.co/docs/transformers/model_doc/ul2)** (from Google Research) released with the paper [Unifying Language Learning Paradigms](https://arxiv.org/abs/2205.05131v1) by Yi Tay, Mostafa Dehghani, Vinh Q. Tran, Xavier Garcia, Dara Bahri, Tal Schuster, Huaixiu Steven Zheng, Neil Houlsby, Donald Metzler 1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (माइक्रोसॉफ्ट रिसर्च से) साथ में दिया गया पेपर [UniSpeech: यूनिफाइड स्पीच रिप्रेजेंटेशन लर्निंग विद लेबलेड एंड अनलेबल्ड डेटा](https:/ /arxiv.org/abs/2101.07597) चेंगई वांग, यू वू, याओ कियान, केनिची कुमातानी, शुजी लियू, फुरु वेई, माइकल ज़ेंग, ज़ुएदोंग हुआंग द्वारा। 1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (माइक्रोसॉफ्ट रिसर्च से) कागज के साथ [UNISPEECH-SAT: यूनिवर्सल स्पीच रिप्रेजेंटेशन लर्निंग विद स्पीकर अवेयर प्री-ट्रेनिंग ](https://arxiv.org/abs/2110.05752) सानयुआन चेन, यू वू, चेंग्यी वांग, झेंगयांग चेन, झूओ चेन, शुजी लियू, जियान वू, याओ कियान, फुरु वेई, जिन्यु ली, जियांगज़ान यू द्वारा पोस्ट किया गया। @@ -413,7 +413,7 @@ conda install -c huggingface transformers 1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (माइक्रोसॉफ्ट रिसर्च से) पेपर के साथ जारी किया गया [WavLM: फुल स्टैक के लिए बड़े पैमाने पर स्व-पर्यवेक्षित पूर्व-प्रशिक्षण स्पीच प्रोसेसिंग] (https://arxiv.org/abs/2110.13900) सानयुआन चेन, चेंगयी वांग, झेंगयांग चेन, यू वू, शुजी लियू, ज़ुओ चेन, जिन्यु ली, नाओयुकी कांडा, ताकुया योशियोका, ज़िओंग जिओ, जियान वू, लॉन्ग झोउ, शुओ रेन, यानमिन कियान, याओ कियान, जियान वू, माइकल ज़ेंग, फुरु वेई। 1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (OpenAI से) साथ में कागज [बड़े पैमाने पर कमजोर पर्यवेक्षण के माध्यम से मजबूत भाषण पहचान](https://cdn. openai.com/papers/whisper.pdf) एलेक रैडफोर्ड, जोंग वूक किम, ताओ जू, ग्रेग ब्रॉकमैन, क्रिस्टीन मैकलीवे, इल्या सुत्स्केवर द्वारा। 1. **[X-CLIP](https://huggingface.co/docs/transformers/model_doc/xclip)** (माइक्रोसॉफ्ट रिसर्च से) कागज के साथ [एक्सपैंडिंग लैंग्वेज-इमेज प्रीट्रेन्ड मॉडल फॉर जनरल वीडियो रिकग्निशन](https: //arxiv.org/abs/2208.02816) बोलिन नी, होउवेन पेंग, मिंगाओ चेन, सोंगयांग झांग, गाओफेंग मेंग, जियानलोंग फू, शिमिंग जियांग, हैबिन लिंग द्वारा। -1. **[X-MOD](https://huggingface.co/docs/transformers/main/model_doc/xmod)** (Meta AI से) Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. द्वाराअनुसंधान पत्र [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) के साथ जारी किया गया +1. **[X-MOD](https://huggingface.co/docs/transformers/model_doc/xmod)** (Meta AI से) Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. द्वाराअनुसंधान पत्र [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) के साथ जारी किया गया 1. **[XGLM](https://huggingface.co/docs/transformers/model_doc/xglm)** (From Facebook AI) released with the paper [Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668) by Xi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (फेसबुक से) साथ में पेपर [क्रॉस-लिंगुअल लैंग्वेज मॉडल प्रीट्रेनिंग] (https://arxiv.org/abs/1901.07291) गिलाउम लैम्पल और एलेक्सिस कोनो द्वारा। 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlm-prophetnet)** (माइक्रोसॉफ्ट रिसर्च से) साथ में कागज [ProphetNet: प्रेडिक्टिंग फ्यूचर एन-ग्राम फॉर सीक्वेंस-टू- सीक्वेंस प्री-ट्रेनिंग](https://arxiv.org/abs/2001.04063) यू यान, वीज़ेन क्यूई, येयुन गोंग, दयाहेंग लियू, नान डुआन, जिउशेंग चेन, रुओफ़ेई झांग और मिंग झोउ द्वारा। diff --git a/README_ja.md b/README_ja.md index d08684c0366d..9d4aceed01df 100644 --- a/README_ja.md +++ b/README_ja.md @@ -298,7 +298,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 🤗Transformersは現在、以下のアーキテクチャを提供しています（それぞれのハイレベルな要約は[こちら](https://huggingface.co/docs/transformers/model_summary)を参照してください）: 1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (Google Research and the Toyota Technological Institute at Chicago から) Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut から公開された研究論文: [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942) -1. **[ALIGN](https://huggingface.co/docs/transformers/main/model_doc/align)** (Google Research から) Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. から公開された研究論文 [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) +1. **[ALIGN](https://huggingface.co/docs/transformers/model_doc/align)** (Google Research から) Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. から公開された研究論文 [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) 1. **[AltCLIP](https://huggingface.co/docs/transformers/model_doc/altclip)** (BAAI から) Chen, Zhongzhi and Liu, Guang and Zhang, Bo-Wen and Ye, Fulong and Yang, Qinghong and Wu, Ledell から公開された研究論文: [AltCLIP: Altering the Language Encoder in CLIP for Extended Language Capabilities](https://arxiv.org/abs/2211.06679) 1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (MIT から) Yuan Gong, Yu-An Chung, James Glass から公開された研究論文: [AST: Audio Spectrogram Transformer](https://arxiv.org/abs/2104.01778) 1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (Facebook から) Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer から公開された研究論文: [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/abs/1910.13461) @@ -315,22 +315,22 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (Facebook から) Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston から公開された研究論文: [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (Facebook から) Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston から公開された研究論文: [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 1. **[BLIP](https://huggingface.co/docs/transformers/model_doc/blip)** (Salesforce から) Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi から公開された研究論文: [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) -1. **[BLIP-2](https://huggingface.co/docs/transformers/main/model_doc/blip-2)** (Salesforce から) Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. から公開された研究論文 [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) +1. **[BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2)** (Salesforce から) Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. から公開された研究論文 [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (BigScience workshop から) [BigScience Workshop](https://bigscience.huggingface.co/) から公開されました. 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (Alexa から) Adrian de Wynter and Daniel J. Perry から公開された研究論文: [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) -1. **[BridgeTower](https://huggingface.co/docs/transformers/main/model_doc/bridgetower)** (Harbin Institute of Technology/Microsoft Research Asia/Intel Labs から) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. +1. **[BridgeTower](https://huggingface.co/docs/transformers/model_doc/bridgetower)** (Harbin Institute of Technology/Microsoft Research Asia/Intel Labs から) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (Google Research から) Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel から公開された研究論文: [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) 1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (Inria/Facebook/Sorbonne から) Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot から公開された研究論文: [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) 1. **[CANINE](https://huggingface.co/docs/transformers/model_doc/canine)** (Google Research から) Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting から公開された研究論文: [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) 1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (OFA-Sys から) An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou から公開された研究論文: [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://arxiv.org/abs/2211.01335) -1. **[CLAP](https://huggingface.co/docs/transformers/main/model_doc/clap)** (LAION-AI から) Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. から公開された研究論文 [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (LAION-AI から) Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. から公開された研究論文 [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) 1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (OpenAI から) Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever から公開された研究論文: [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) 1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (University of Göttingen から) Timo Lüddecke and Alexander Ecker から公開された研究論文: [Image Segmentation Using Text and Image Prompts](https://arxiv.org/abs/2112.10003) 1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (Salesforce から) Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong から公開された研究論文: [A Conversational Paradigm for Program Synthesis](https://arxiv.org/abs/2203.13474) 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (Microsoft Research Asia から) Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang から公開された研究論文: [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (YituTech から) Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan から公開された研究論文: [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (Facebook AI から) Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie から公開された研究論文: [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) -1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (Tsinghua University から) Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun から公開された研究論文: [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (Salesforce から) Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher から公開された研究論文: [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (Microsoft から) Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang から公開された研究論文: [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) @@ -340,7 +340,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (Berkeley/Facebook/Google から) Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch から公開された研究論文: [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://arxiv.org/abs/2106.01345) 1. **[Deformable DETR](https://huggingface.co/docs/transformers/model_doc/deformable_detr)** (SenseTime Research から) Xizhou Zhu, Weijie Su, Lewei Lu, Bin Li, Xiaogang Wang, Jifeng Dai から公開された研究論文: [Deformable DETR: Deformable Transformers for End-to-End Object Detection](https://arxiv.org/abs/2010.04159) 1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (Facebook から) Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou から公開された研究論文: [Training data-efficient image transformers & distillation through attention](https://arxiv.org/abs/2012.12877) -1. **[DETA](https://huggingface.co/docs/transformers/main/model_doc/deta)** (The University of Texas at Austin から) Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. から公開された研究論文 [NMS Strikes Back](https://arxiv.org/abs/2212.06137) +1. **[DETA](https://huggingface.co/docs/transformers/model_doc/deta)** (The University of Texas at Austin から) Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. から公開された研究論文 [NMS Strikes Back](https://arxiv.org/abs/2212.06137) 1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (Facebook から) Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko から公開された研究論文: [End-to-End Object Detection with Transformers](https://arxiv.org/abs/2005.12872) 1. **[DialoGPT](https://huggingface.co/docs/transformers/model_doc/dialogpt)** (Microsoft Research から) Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan から公開された研究論文: [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) 1. **[DiNAT](https://huggingface.co/docs/transformers/model_doc/dinat)** (SHI Labs から) Ali Hassani and Humphrey Shi から公開された研究論文: [Dilated Neighborhood Attention Transformer](https://arxiv.org/abs/2209.15001) @@ -350,11 +350,11 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[DPR](https://huggingface.co/docs/transformers/model_doc/dpr)** (Facebook から) Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih から公開された研究論文: [Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) 1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (Intel Labs から) René Ranftl, Alexey Bochkovskiy, Vladlen Koltun から公開された研究論文: [Vision Transformers for Dense Prediction](https://arxiv.org/abs/2103.13413) 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (Snap Research から) Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. から公開された研究論文 [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) -1. **[EfficientNet](https://huggingface.co/docs/transformers/main/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (Google Research/Stanford University から) Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning から公開された研究論文: [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (Google Research から) Sascha Rothe, Shashi Narayan, Aliaksei Severyn から公開された研究論文: [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (Baidu から) Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu から公開された研究論文: [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) -1. **[ErnieM](https://huggingface.co/docs/transformers/main/model_doc/ernie_m)** (Baidu から) Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. から公開された研究論文 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) +1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (Baidu から) Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. から公開された研究論文 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) 1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (Meta AI から) はトランスフォーマープロテイン言語モデルです. **ESM-1b** は Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus から公開された研究論文: [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118). **ESM-1v** は Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives　から公開された研究論文: [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648). **ESM-2** と　**ESMFold** は Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives から公開された研究論文: [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) 1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (Google AI から) Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V から公開されたレポジトリー [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) Le, and Jason Wei 1. **[FLAN-UL2](https://huggingface.co/docs/transformers/model_doc/flan-ul2)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei @@ -371,13 +371,13 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (OpenAI から) Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever** から公開された研究論文: [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) 1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (EleutherAI から) Ben Wang and Aran Komatsuzaki から公開されたレポジトリー [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) 1. **[GPT-Sw3](https://huggingface.co/docs/transformers/model_doc/gpt-sw3)** (AI-Sweden から) Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman, Fredrik Carlsson, Magnus Sahlgren から公開された研究論文: [Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish](http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf) -1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/main/model_doc/gptsan-japanese)** [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) 坂本俊之(tanreinama)からリリースされました. +1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/model_doc/gptsan-japanese)** [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) 坂本俊之(tanreinama)からリリースされました. 1. **[Graphormer](https://huggingface.co/docs/transformers/model_doc/graphormer)** (Microsoft から) Chengxuan Ying, Tianle Cai, Shengjie Luo, Shuxin Zheng, Guolin Ke, Di He, Yanming Shen, Tie-Yan Liu から公開された研究論文: [Do Transformers Really Perform Bad for Graph Representation?](https://arxiv.org/abs/2106.05234). 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (UCSD, NVIDIA から) Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang から公開された研究論文: [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://arxiv.org/abs/2202.11094) 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (Facebook から) Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed から公開された研究論文: [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://arxiv.org/abs/2106.07447) 1. **[I-BERT](https://huggingface.co/docs/transformers/model_doc/ibert)** (Berkeley から) Sehoon Kim, Amir Gholami, Zhewei Yao, Michael W. Mahoney, Kurt Keutzer から公開された研究論文: [I-BERT: Integer-only BERT Quantization](https://arxiv.org/abs/2101.01321) 1. **[ImageGPT](https://huggingface.co/docs/transformers/model_doc/imagegpt)** (OpenAI から) Mark Chen, Alec Radford, Rewon Child, Jeffrey Wu, Heewoo Jun, David Luan, Ilya Sutskever から公開された研究論文: [Generative Pretraining from Pixels](https://openai.com/blog/image-gpt/) -1. **[Informer](https://huggingface.co/docs/transformers/main/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. +1. **[Informer](https://huggingface.co/docs/transformers/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. 1. **[Jukebox](https://huggingface.co/docs/transformers/model_doc/jukebox)** (OpenAI から) Prafulla Dhariwal, Heewoo Jun, Christine Payne, Jong Wook Kim, Alec Radford, Ilya Sutskever から公開された研究論文: [Jukebox: A Generative Model for Music](https://arxiv.org/pdf/2005.00341.pdf) 1. **[LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm)** (Microsoft Research Asia から) Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou から公開された研究論文: [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) 1. **[LayoutLMv2](https://huggingface.co/docs/transformers/model_doc/layoutlmv2)** (Microsoft Research Asia から) Yang Xu, Yiheng Xu, Tengchao Lv, Lei Cui, Furu Wei, Guoxin Wang, Yijuan Lu, Dinei Florencio, Cha Zhang, Wanxiang Che, Min Zhang, Lidong Zhou から公開された研究論文: [LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding](https://arxiv.org/abs/2012.14740) @@ -400,7 +400,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook から) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan から公開された研究論文: [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) -1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (Alibaba Research から) Peng Wang, Cheng Da, and Cong Yao. から公開された研究論文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research から) Peng Wang, Cheng Da, and Cong Yao. から公開された研究論文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (Studio Ousia から) Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka から公開された研究論文: [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (CMU/Google Brain から) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou から公開された研究論文: [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (Google Inc. から) Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam から公開された研究論文: [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) @@ -437,7 +437,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (NVIDIA から) Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo から公開された研究論文: [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://arxiv.org/abs/2105.15203) 1. **[SEW](https://huggingface.co/docs/transformers/model_doc/sew)** (ASAPP から) Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi から公開された研究論文: [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) 1. **[SEW-D](https://huggingface.co/docs/transformers/model_doc/sew_d)** (ASAPP から) Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi から公開された研究論文: [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) -1. **[SpeechT5](https://huggingface.co/docs/transformers/main/model_doc/speecht5)** (Microsoft Research から) Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. から公開された研究論文 [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (Microsoft Research から) Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. から公開された研究論文 [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) 1. **[SpeechToTextTransformer](https://huggingface.co/docs/transformers/model_doc/speech_to_text)** (Facebook から), Changhan Wang, Yun Tang, Xutai Ma, Anne Wu, Dmytro Okhonko, Juan Pino から公開された研究論文: [fairseq S2T: Fast Speech-to-Text Modeling with fairseq](https://arxiv.org/abs/2010.05171) 1. **[SpeechToTextTransformer2](https://huggingface.co/docs/transformers/model_doc/speech_to_text_2)** (Facebook から), Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli, Alexis Conneau から公開された研究論文: [Large-Scale Self- and Semi-Supervised Learning for Speech Translation](https://arxiv.org/abs/2104.06678) 1. **[Splinter](https://huggingface.co/docs/transformers/model_doc/splinter)** (Tel Aviv University から), Ori Ram, Yuval Kirstain, Jonathan Berant, Amir Globerson, Omer Levy から公開された研究論文: [Few-Shot Question Answering by Pretraining Span Selection](https://arxiv.org/abs/2101.00438) @@ -456,7 +456,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[Trajectory Transformer](https://huggingface.co/docs/transformers/model_doc/trajectory_transformers)** (the University of California at Berkeley から) Michael Janner, Qiyang Li, Sergey Levine から公開された研究論文: [Offline Reinforcement Learning as One Big Sequence Modeling Problem](https://arxiv.org/abs/2106.02039) 1. **[Transformer-XL](https://huggingface.co/docs/transformers/model_doc/transfo-xl)** (Google/CMU から) Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov から公開された研究論文: [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) 1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (Microsoft から), Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei から公開された研究論文: [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) -1. **[TVLT](https://huggingface.co/docs/transformers/main/model_doc/tvlt)** (from UNC Chapel Hill から), Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal から公開された研究論文: [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) +1. **[TVLT](https://huggingface.co/docs/transformers/model_doc/tvlt)** (from UNC Chapel Hill から), Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal から公開された研究論文: [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) 1. **[UL2](https://huggingface.co/docs/transformers/model_doc/ul2)** (Google Research から) Yi Tay, Mostafa Dehghani, Vinh Q から公開された研究論文: [Unifying Language Learning Paradigms](https://arxiv.org/abs/2205.05131v1) Tran, Xavier Garcia, Dara Bahri, Tal Schuster, Huaixiu Steven Zheng, Neil Houlsby, Donald Metzler 1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (Microsoft Research から) Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang から公開された研究論文: [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) 1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (Microsoft Research から) Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu から公開された研究論文: [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://arxiv.org/abs/2110.05752) @@ -475,7 +475,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (Microsoft Research から) Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei から公開された研究論文: [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) 1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (OpenAI から) Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever から公開された研究論文: [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) 1. **[X-CLIP](https://huggingface.co/docs/transformers/model_doc/xclip)** (Microsoft Research から) Bolin Ni, Houwen Peng, Minghao Chen, Songyang Zhang, Gaofeng Meng, Jianlong Fu, Shiming Xiang, Haibin Ling から公開された研究論文: [Expanding Language-Image Pretrained Models for General Video Recognition](https://arxiv.org/abs/2208.02816) -1. **[X-MOD](https://huggingface.co/docs/transformers/main/model_doc/xmod)** (Meta AI から) Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. から公開された研究論文 [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) +1. **[X-MOD](https://huggingface.co/docs/transformers/model_doc/xmod)** (Meta AI から) Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. から公開された研究論文 [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) 1. **[XGLM](https://huggingface.co/docs/transformers/model_doc/xglm)** (From Facebook AI) Xi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li から公開された研究論文: [Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668) 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (Facebook から) Guillaume Lample and Alexis Conneau から公開された研究論文: [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlm-prophetnet)** (Microsoft Research から) Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou から公開された研究論文: [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) diff --git a/README_ko.md b/README_ko.md index d68a9b54e2d3..b8e69fccfa6b 100644 --- a/README_ko.md +++ b/README_ko.md @@ -213,7 +213,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 🤗 Transformers는 다음 모델들을 제공합니다 (각 모델의 요약은 [여기](https://huggingface.co/docs/transformers/model_summary)서 확인하세요): 1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. -1. **[ALIGN](https://huggingface.co/docs/transformers/main/model_doc/align)** (Google Research 에서 제공)은 Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig.의 [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918)논문과 함께 발표했습니다. +1. **[ALIGN](https://huggingface.co/docs/transformers/model_doc/align)** (Google Research 에서 제공)은 Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig.의 [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918)논문과 함께 발표했습니다. 1. **[AltCLIP](https://huggingface.co/docs/transformers/model_doc/altclip)** (from BAAI) released with the paper [AltCLIP: Altering the Language Encoder in CLIP for Extended Language Capabilities](https://arxiv.org/abs/2211.06679) by Chen, Zhongzhi and Liu, Guang and Zhang, Bo-Wen and Ye, Fulong and Yang, Qinghong and Wu, Ledell. 1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://arxiv.org/abs/2104.01778) by Yuan Gong, Yu-An Chung, James Glass. 1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/pdf/1910.13461.pdf) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer. @@ -230,22 +230,22 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BLIP](https://huggingface.co/docs/transformers/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. -1. **[BLIP-2](https://huggingface.co/docs/transformers/main/model_doc/blip-2)** (Salesforce 에서 제공)은 Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi.의 [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597)논문과 함께 발표했습니다. +1. **[BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2)** (Salesforce 에서 제공)은 Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi.의 [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597)논문과 함께 발표했습니다. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (Alexa 에서) Adrian de Wynter and Daniel J. Perry 의 [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) 논문과 함께 발표했습니다. -1. **[BridgeTower](https://huggingface.co/docs/transformers/main/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. +1. **[BridgeTower](https://huggingface.co/docs/transformers/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (Google Research 에서) Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel 의 [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) 논문과 함께 발표했습니다. 1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (Inria/Facebook/Sorbonne 에서) Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot 의 [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) 논문과 함께 발표했습니다. 1. **[CANINE](https://huggingface.co/docs/transformers/model_doc/canine)** (Google Research 에서) Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting 의 [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) 논문과 함께 발표했습니다. 1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (OFA-Sys 에서) An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou 의 [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://arxiv.org/abs/2211.01335) 논문과 함께 발표했습니다. -1. **[CLAP](https://huggingface.co/docs/transformers/main/model_doc/clap)** (LAION-AI 에서 제공)은 Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov.의 [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687)논문과 함께 발표했습니다. +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (LAION-AI 에서 제공)은 Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov.의 [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687)논문과 함께 발표했습니다. 1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (OpenAI 에서) Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever 의 [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) 논문과 함께 발표했습니다. 1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (University of Göttingen 에서) Timo Lüddecke and Alexander Ecker 의 [Image Segmentation Using Text and Image Prompts](https://arxiv.org/abs/2112.10003) 논문과 함께 발표했습니다. 1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (Salesforce 에서) Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong 의 [A Conversational Paradigm for Program Synthesis](https://arxiv.org/abs/2203.13474) 논문과 함께 발표했습니다. 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (Microsoft Research Asia 에서) Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang 의 [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) 논문과 함께 발표했습니다. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (YituTech 에서) Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan 의 [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) 논문과 함께 발표했습니다. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (Facebook AI 에서) Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie 의 [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) 논문과 함께 발표했습니다. -1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (Tsinghua University 에서) Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun 의 [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) 논문과 함께 발표했습니다. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (Salesforce 에서) Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher 의 [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) 논문과 함께 발표했습니다. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (Microsoft 에서) Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang 의 [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) 논문과 함께 발표했습니다. @@ -255,7 +255,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (Berkeley/Facebook/Google 에서) Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch 의 [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://arxiv.org/abs/2106.01345) 논문과 함께 발표했습니다. 1. **[Deformable DETR](https://huggingface.co/docs/transformers/model_doc/deformable_detr)** (SenseTime Research 에서) Xizhou Zhu, Weijie Su, Lewei Lu, Bin Li, Xiaogang Wang, Jifeng Dai 의 [Deformable DETR: Deformable Transformers for End-to-End Object Detection](https://arxiv.org/abs/2010.04159) 논문과 함께 발표했습니다. 1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (Facebook 에서) Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou 의 [Training data-efficient image transformers & distillation through attention](https://arxiv.org/abs/2012.12877) 논문과 함께 발표했습니다. -1. **[DETA](https://huggingface.co/docs/transformers/main/model_doc/deta)** (The University of Texas at Austin 에서 제공)은 Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl.의 [NMS Strikes Back](https://arxiv.org/abs/2212.06137)논문과 함께 발표했습니다. +1. **[DETA](https://huggingface.co/docs/transformers/model_doc/deta)** (The University of Texas at Austin 에서 제공)은 Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl.의 [NMS Strikes Back](https://arxiv.org/abs/2212.06137)논문과 함께 발표했습니다. 1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (Facebook 에서) Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko 의 [End-to-End Object Detection with Transformers](https://arxiv.org/abs/2005.12872) 논문과 함께 발표했습니다. 1. **[DialoGPT](https://huggingface.co/docs/transformers/model_doc/dialogpt)** (Microsoft Research 에서) Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan 의 [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) 논문과 함께 발표했습니다. 1. **[DiNAT](https://huggingface.co/docs/transformers/model_doc/dinat)** (SHI Labs 에서) Ali Hassani and Humphrey Shi 의 [Dilated Neighborhood Attention Transformer](https://arxiv.org/abs/2209.15001) 논문과 함께 발표했습니다. @@ -265,11 +265,11 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[DPR](https://huggingface.co/docs/transformers/model_doc/dpr)** (Facebook 에서) Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih 의 [Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) 논문과 함께 발표했습니다. 1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (Intel Labs 에서) René Ranftl, Alexey Bochkovskiy, Vladlen Koltun 의 [Vision Transformers for Dense Prediction](https://arxiv.org/abs/2103.13413) 논문과 함께 발표했습니다. 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. -1. **[EfficientNet](https://huggingface.co/docs/transformers/main/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (Google Research/Stanford University 에서) Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning 의 [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) 논문과 함께 발표했습니다. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (Google Research 에서) Sascha Rothe, Shashi Narayan, Aliaksei Severyn 의 [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) 논문과 함께 발표했습니다. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (Baidu 에서) Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu 의 [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) 논문과 함께 발표했습니다. -1. **[ErnieM](https://huggingface.co/docs/transformers/main/model_doc/ernie_m)** (Baidu 에서 제공)은 Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang.의 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674)논문과 함께 발표했습니다. +1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (Baidu 에서 제공)은 Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang.의 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674)논문과 함께 발표했습니다. 1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2** was released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives. 1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei 1. **[FLAN-UL2](https://huggingface.co/docs/transformers/model_doc/flan-ul2)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei @@ -286,13 +286,13 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (OpenAI 에서) Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever** 의 [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) 논문과 함께 발표했습니다. 1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released in the repository [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki. 1. **[GPT-Sw3](https://huggingface.co/docs/transformers/model_doc/gpt-sw3)** (AI-Sweden 에서) Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman, Fredrik Carlsson, Magnus Sahlgren. 의 [Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish](http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf) 논문과 함께 발표했습니다. -1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/main/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). +1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by Toshiyuki Sakamoto(tanreinama). 1. **[Graphormer](https://huggingface.co/docs/transformers/model_doc/graphormer)** (from Microsoft) Chengxuan Ying, Tianle Cai, Shengjie Luo, Shuxin Zheng, Guolin Ke, Di He, Yanming Shen, Tie-Yan Liu 의 [Do Transformers Really Perform Bad for Graph Representation?](https://arxiv.org/abs/2106.05234) 논문과 함께 발표했습니다. 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (UCSD, NVIDIA 에서) Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang 의 [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://arxiv.org/abs/2202.11094) 논문과 함께 발표했습니다. 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (Facebook 에서) Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed 의 [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://arxiv.org/abs/2106.07447) 논문과 함께 발표했습니다. 1. **[I-BERT](https://huggingface.co/docs/transformers/model_doc/ibert)** (Berkeley 에서) Sehoon Kim, Amir Gholami, Zhewei Yao, Michael W. Mahoney, Kurt Keutzer 의 [I-BERT: Integer-only BERT Quantization](https://arxiv.org/abs/2101.01321) 논문과 함께 발표했습니다. 1. **[ImageGPT](https://huggingface.co/docs/transformers/model_doc/imagegpt)** (OpenAI 에서) Mark Chen, Alec Radford, Rewon Child, Jeffrey Wu, Heewoo Jun, David Luan, Ilya Sutskever 의 [Generative Pretraining from Pixels](https://openai.com/blog/image-gpt/) 논문과 함께 발표했습니다. -1. **[Informer](https://huggingface.co/docs/transformers/main/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. +1. **[Informer](https://huggingface.co/docs/transformers/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. 1. **[Jukebox](https://huggingface.co/docs/transformers/model_doc/jukebox)** (OpenAI 에서) Prafulla Dhariwal, Heewoo Jun, Christine Payne, Jong Wook Kim, Alec Radford, Ilya Sutskever 의 [Jukebox: A Generative Model for Music](https://arxiv.org/pdf/2005.00341.pdf) 논문과 함께 발표했습니다. 1. **[LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm)** (Microsoft Research Asia 에서) Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou 의 [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) 논문과 함께 발표했습니다. 1. **[LayoutLMv2](https://huggingface.co/docs/transformers/model_doc/layoutlmv2)** (Microsoft Research Asia 에서) Yang Xu, Yiheng Xu, Tengchao Lv, Lei Cui, Furu Wei, Guoxin Wang, Yijuan Lu, Dinei Florencio, Cha Zhang, Wanxiang Che, Min Zhang, Lidong Zhou 의 [LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding](https://arxiv.org/abs/2012.14740) 논문과 함께 발표했습니다. @@ -315,7 +315,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook 에서) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 의 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 논문과 함께 발표했습니다. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. -1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (Alibaba Research 에서 제공)은 Peng Wang, Cheng Da, and Cong Yao.의 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592)논문과 함께 발표했습니다. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research 에서 제공)은 Peng Wang, Cheng Da, and Cong Yao.의 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592)논문과 함께 발표했습니다. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (Studio Ousia 에서) Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka 의 [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 논문과 함께 발표했습니다. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (CMU/Google Brain 에서) Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou 의 [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 논문과 함께 발표했습니다. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (Google Inc. 에서) Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam 의 [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) 논문과 함께 발표했습니다. @@ -352,7 +352,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (NVIDIA 에서) Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo 의 [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://arxiv.org/abs/2105.15203) 논문과 함께 발표했습니다. 1. **[SEW](https://huggingface.co/docs/transformers/model_doc/sew)** (ASAPP 에서) Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi 의 [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) 논문과 함께 발표했습니다. 1. **[SEW-D](https://huggingface.co/docs/transformers/model_doc/sew_d)** (ASAPP 에서) Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi 의 [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) 논문과 함께 발표했습니다. -1. **[SpeechT5](https://huggingface.co/docs/transformers/main/model_doc/speecht5)** (Microsoft Research 에서 제공)은 Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei.의 [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205)논문과 함께 발표했습니다. +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (Microsoft Research 에서 제공)은 Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei.의 [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205)논문과 함께 발표했습니다. 1. **[SpeechToTextTransformer](https://huggingface.co/docs/transformers/model_doc/speech_to_text)** (Facebook 에서) Changhan Wang, Yun Tang, Xutai Ma, Anne Wu, Dmytro Okhonko, Juan Pino 의 [fairseq S2T: Fast Speech-to-Text Modeling with fairseq](https://arxiv.org/abs/2010.05171) 논문과 함께 발표했습니다. 1. **[SpeechToTextTransformer2](https://huggingface.co/docs/transformers/model_doc/speech_to_text_2)** (Facebook 에서) Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli, Alexis Conneau 의 [Large-Scale Self- and Semi-Supervised Learning for Speech Translation](https://arxiv.org/abs/2104.06678) 논문과 함께 발표했습니다. 1. **[Splinter](https://huggingface.co/docs/transformers/model_doc/splinter)** (Tel Aviv University 에서) Ori Ram, Yuval Kirstain, Jonathan Berant, Amir Globerson, Omer Levy 의 [Few-Shot Question Answering by Pretraining Span Selection](https://arxiv.org/abs/2101.00438) 논문과 함께 발표했습니다. @@ -371,7 +371,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[Trajectory Transformer](https://huggingface.co/docs/transformers/model_doc/trajectory_transformers)** (the University of California at Berkeley 에서) Michael Janner, Qiyang Li, Sergey Levin 의 [Offline Reinforcement Learning as One Big Sequence Modeling Problem](https://arxiv.org/abs/2106.02039) 논문과 함께 발표했습니다. 1. **[Transformer-XL](https://huggingface.co/docs/transformers/model_doc/transfo-xl)** (Google/CMU 에서) Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov 의 [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) 논문과 함께 발표했습니다. 1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (Microsoft 에서) Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei 의 [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) 논문과 함께 발표했습니다. -1. **[TVLT](https://huggingface.co/docs/transformers/main/model_doc/tvlt)** (from UNC Chapel Hill 에서) Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal 의 [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) 논문과 함께 발표했습니다. +1. **[TVLT](https://huggingface.co/docs/transformers/model_doc/tvlt)** (from UNC Chapel Hill 에서) Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal 의 [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) 논문과 함께 발표했습니다. 1. **[UL2](https://huggingface.co/docs/transformers/model_doc/ul2)** (Google Research 에서) Yi Tay, Mostafa Dehghani, Vinh Q. Tran, Xavier Garcia, Dara Bahri, Tal Schuster, Huaixiu Steven Zheng, Neil Houlsby, Donald Metzle 의 [Unifying Language Learning Paradigms](https://arxiv.org/abs/2205.05131v1) 논문과 함께 발표했습니다. 1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (Microsoft Research 에서) Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang 의 [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) 논문과 함께 발표했습니다. 1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (Microsoft Research 에서) Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu 의 [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://arxiv.org/abs/2110.05752) 논문과 함께 발표했습니다. @@ -390,7 +390,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (Microsoft Research 에서) Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei 의 [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) 논문과 함께 발표했습니다. 1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (OpenAI 에서) Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever 의 [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) 논문과 함께 발표했습니다. 1. **[X-CLIP](https://huggingface.co/docs/transformers/model_doc/xclip)** (Microsoft Research 에서) Bolin Ni, Houwen Peng, Minghao Chen, Songyang Zhang, Gaofeng Meng, Jianlong Fu, Shiming Xiang, Haibin Ling 의 [Expanding Language-Image Pretrained Models for General Video Recognition](https://arxiv.org/abs/2208.02816) 논문과 함께 발표했습니다. -1. **[X-MOD](https://huggingface.co/docs/transformers/main/model_doc/xmod)** (Meta AI 에서 제공)은 Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe.의 [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255)논문과 함께 발표했습니다. +1. **[X-MOD](https://huggingface.co/docs/transformers/model_doc/xmod)** (Meta AI 에서 제공)은 Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe.의 [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255)논문과 함께 발표했습니다. 1. **[XGLM](https://huggingface.co/docs/transformers/model_doc/xglm)** (Facebook AI 에서 제공) Xi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li 의 [Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668) 논문과 함께 발표했습니다. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (Facebook 에서) Guillaume Lample and Alexis Conneau 의 [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) 논문과 함께 발표했습니다. 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlm-prophetnet)** (Microsoft Research 에서) Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou 의 [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) 논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index 2e8f9464e06c..8527a522949b 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -237,7 +237,7 @@ conda install -c huggingface transformers 🤗 Transformers 目前支持如下的架构（模型概述请阅[这里](https://huggingface.co/docs/transformers/model_summary)）： 1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (来自 Google Research and the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。 -1. **[ALIGN](https://huggingface.co/docs/transformers/main/model_doc/align)** (来自 Google Research) 伴随论文 [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) 由 Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig 发布。 +1. **[ALIGN](https://huggingface.co/docs/transformers/model_doc/align)** (来自 Google Research) 伴随论文 [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) 由 Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig 发布。 1. **[AltCLIP](https://huggingface.co/docs/transformers/model_doc/altclip)** (来自 BAAI) 伴随论文 [AltCLIP: Altering the Language Encoder in CLIP for Extended Language Capabilities](https://arxiv.org/abs/2211.06679) 由 Chen, Zhongzhi and Liu, Guang and Zhang, Bo-Wen and Ye, Fulong and Yang, Qinghong and Wu, Ledell 发布。 1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (来自 MIT) 伴随论文 [AST: Audio Spectrogram Transformer](https://arxiv.org/abs/2104.01778) 由 Yuan Gong, Yu-An Chung, James Glass 发布。 1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (来自 Facebook) 伴随论文 [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/pdf/1910.13461.pdf) 由 Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer 发布。 @@ -254,22 +254,22 @@ conda install -c huggingface transformers 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (来自 Facebook) 伴随论文 [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 由 Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston 发布。 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (来自 Facebook) 伴随论文 [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 由 Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston 发布。 1. **[BLIP](https://huggingface.co/docs/transformers/model_doc/blip)** (来自 Salesforce) 伴随论文 [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) 由 Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi 发布。 -1. **[BLIP-2](https://huggingface.co/docs/transformers/main/model_doc/blip-2)** (来自 Salesforce) 伴随论文 [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) 由 Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi 发布。 +1. **[BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2)** (来自 Salesforce) 伴随论文 [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) 由 Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi 发布。 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (来自 Alexa) 伴随论文 [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) 由 Adrian de Wynter and Daniel J. Perry 发布。 -1. **[BridgeTower](https://huggingface.co/docs/transformers/main/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. +1. **[BridgeTower](https://huggingface.co/docs/transformers/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (来自 Google Research) 伴随论文 [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) 由 Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel 发布。 1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (来自 Inria/Facebook/Sorbonne) 伴随论文 [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) 由 Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot 发布。 1. **[CANINE](https://huggingface.co/docs/transformers/model_doc/canine)** (来自 Google Research) 伴随论文 [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) 由 Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting 发布。 1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (来自 OFA-Sys) 伴随论文 [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://arxiv.org/abs/2211.01335) 由 An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou 发布。 -1. **[CLAP](https://huggingface.co/docs/transformers/main/model_doc/clap)** (来自 LAION-AI) 伴随论文 [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) 由 Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov 发布。 +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (来自 LAION-AI) 伴随论文 [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) 由 Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov 发布。 1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (来自 OpenAI) 伴随论文 [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) 由 Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever 发布。 1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (来自 University of Göttingen) 伴随论文 [Image Segmentation Using Text and Image Prompts](https://arxiv.org/abs/2112.10003) 由 Timo Lüddecke and Alexander Ecker 发布。 1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (来自 Salesforce) 伴随论文 [A Conversational Paradigm for Program Synthesis](https://arxiv.org/abs/2203.13474) 由 Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong 发布。 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (来自 Microsoft Research Asia) 伴随论文 [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) 由 Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang 发布。 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (来自 YituTech) 伴随论文 [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) 由 Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan 发布。 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (来自 Facebook AI) 伴随论文 [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) 由 Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie 发布。 -1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (来自 Tsinghua University) 伴随论文 [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) 由 Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun 发布。 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (来自 Salesforce) 伴随论文 [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) 由 Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher 发布。 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (来自 Microsoft) 伴随论文 [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) 由 Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang 发布。 @@ -279,7 +279,7 @@ conda install -c huggingface transformers 1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (来自 Berkeley/Facebook/Google) 伴随论文 [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://arxiv.org/abs/2106.01345) 由 Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch 发布。 1. **[Deformable DETR](https://huggingface.co/docs/transformers/model_doc/deformable_detr)** (来自 SenseTime Research) 伴随论文 [Deformable DETR: Deformable Transformers for End-to-End Object Detection](https://arxiv.org/abs/2010.04159) 由 Xizhou Zhu, Weijie Su, Lewei Lu, Bin Li, Xiaogang Wang, Jifeng Dai 发布。 1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (来自 Facebook) 伴随论文 [Training data-efficient image transformers & distillation through attention](https://arxiv.org/abs/2012.12877) 由 Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou 发布。 -1. **[DETA](https://huggingface.co/docs/transformers/main/model_doc/deta)** (来自 The University of Texas at Austin) 伴随论文 [NMS Strikes Back](https://arxiv.org/abs/2212.06137) 由 Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl 发布。 +1. **[DETA](https://huggingface.co/docs/transformers/model_doc/deta)** (来自 The University of Texas at Austin) 伴随论文 [NMS Strikes Back](https://arxiv.org/abs/2212.06137) 由 Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl 发布。 1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (来自 Facebook) 伴随论文 [End-to-End Object Detection with Transformers](https://arxiv.org/abs/2005.12872) 由 Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko 发布。 1. **[DialoGPT](https://huggingface.co/docs/transformers/model_doc/dialogpt)** (来自 Microsoft Research) 伴随论文 [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) 由 Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan 发布。 1. **[DiNAT](https://huggingface.co/docs/transformers/model_doc/dinat)** (来自 SHI Labs) 伴随论文 [Dilated Neighborhood Attention Transformer](https://arxiv.org/abs/2209.15001) 由 Ali Hassani and Humphrey Shi 发布。 @@ -289,11 +289,11 @@ conda install -c huggingface transformers 1. **[DPR](https://huggingface.co/docs/transformers/model_doc/dpr)** (来自 Facebook) 伴随论文 [Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) 由 Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih 发布。 1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (来自 Intel Labs) 伴随论文 [Vision Transformers for Dense Prediction](https://arxiv.org/abs/2103.13413) 由 René Ranftl, Alexey Bochkovskiy, Vladlen Koltun 发布。 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (来自 Snap Research) 伴随论文 [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) 由 Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren 发布。 -1. **[EfficientNet](https://huggingface.co/docs/transformers/main/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (来自 Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning 发布。 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (来自 Google Research) 伴随论文 [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) 由 Sascha Rothe, Shashi Narayan, Aliaksei Severyn 发布。 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (来自 Baidu) 伴随论文 [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu 发布。 -1. **[ErnieM](https://huggingface.co/docs/transformers/main/model_doc/ernie_m)** (来自 Baidu) 伴随论文 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) 由 Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang 发布。 +1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (来自 Baidu) 伴随论文 [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) 由 Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang 发布。 1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2** was released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives. 1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei 1. **[FLAN-UL2](https://huggingface.co/docs/transformers/model_doc/flan-ul2)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei @@ -310,13 +310,13 @@ conda install -c huggingface transformers 1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (来自 OpenAI) 伴随论文 [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) 由 Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever** 发布。 1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (来自 EleutherAI) 伴随论文 [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) 由 Ben Wang and Aran Komatsuzaki 发布。 1. **[GPT-Sw3](https://huggingface.co/docs/transformers/model_doc/gpt-sw3)** (from AI-Sweden) released with the paper [Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish](http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf) by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman, Fredrik Carlsson, Magnus Sahlgren. -1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/main/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by 坂本俊之(tanreinama). +1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by 坂本俊之(tanreinama). 1. **[Graphormer](https://huggingface.co/docs/transformers/model_doc/graphormer)** (from Microsoft) released with the paper [Do Transformers Really Perform Bad for Graph Representation?](https://arxiv.org/abs/2106.05234) by Chengxuan Ying, Tianle Cai, Shengjie Luo, Shuxin Zheng, Guolin Ke, Di He, Yanming Shen, Tie-Yan Liu. 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (来自 UCSD, NVIDIA) 伴随论文 [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://arxiv.org/abs/2202.11094) 由 Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang 发布。 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (来自 Facebook) 伴随论文 [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://arxiv.org/abs/2106.07447) 由 Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed 发布。 1. **[I-BERT](https://huggingface.co/docs/transformers/model_doc/ibert)** (来自 Berkeley) 伴随论文 [I-BERT: Integer-only BERT Quantization](https://arxiv.org/abs/2101.01321) 由 Sehoon Kim, Amir Gholami, Zhewei Yao, Michael W. Mahoney, Kurt Keutzer 发布。 1. **[ImageGPT](https://huggingface.co/docs/transformers/model_doc/imagegpt)** (来自 OpenAI) 伴随论文 [Generative Pretraining from Pixels](https://openai.com/blog/image-gpt/) 由 Mark Chen, Alec Radford, Rewon Child, Jeffrey Wu, Heewoo Jun, David Luan, Ilya Sutskever 发布。 -1. **[Informer](https://huggingface.co/docs/transformers/main/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. +1. **[Informer](https://huggingface.co/docs/transformers/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. 1. **[Jukebox](https://huggingface.co/docs/transformers/model_doc/jukebox)** (from OpenAI) released with the paper [Jukebox: A Generative Model for Music](https://arxiv.org/pdf/2005.00341.pdf) by Prafulla Dhariwal, Heewoo Jun, Christine Payne, Jong Wook Kim, Alec Radford, Ilya Sutskever. 1. **[LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm)** (来自 Microsoft Research Asia) 伴随论文 [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) 由 Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou 发布。 1. **[LayoutLMv2](https://huggingface.co/docs/transformers/model_doc/layoutlmv2)** (来自 Microsoft Research Asia) 伴随论文 [LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding](https://arxiv.org/abs/2012.14740) 由 Yang Xu, Yiheng Xu, Tengchao Lv, Lei Cui, Furu Wei, Guoxin Wang, Yijuan Lu, Dinei Florencio, Cha Zhang, Wanxiang Che, Min Zhang, Lidong Zhou 发布。 @@ -339,7 +339,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (来自 Facebook) 伴随论文 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 由 Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 发布。 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 -1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (来自 Alibaba Research) 伴随论文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 由 Peng Wang, Cheng Da, and Cong Yao 发布。 +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (来自 Alibaba Research) 伴随论文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 由 Peng Wang, Cheng Da, and Cong Yao 发布。 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (来自 Studio Ousia) 伴随论文 [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) 由 Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka 发布。 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (来自 CMU/Google Brain) 伴随论文 [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) 由 Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou 发布。 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (来自 Google Inc.) 伴随论文 [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) 由 Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam 发布。 @@ -376,7 +376,7 @@ conda install -c huggingface transformers 1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (来自 NVIDIA) 伴随论文 [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://arxiv.org/abs/2105.15203) 由 Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo 发布。 1. **[SEW](https://huggingface.co/docs/transformers/model_doc/sew)** (来自 ASAPP) 伴随论文 [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) 由 Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi 发布。 1. **[SEW-D](https://huggingface.co/docs/transformers/model_doc/sew_d)** (来自 ASAPP) 伴随论文 [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) 由 Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi 发布。 -1. **[SpeechT5](https://huggingface.co/docs/transformers/main/model_doc/speecht5)** (来自 Microsoft Research) 伴随论文 [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) 由 Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei 发布。 +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (来自 Microsoft Research) 伴随论文 [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) 由 Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei 发布。 1. **[SpeechToTextTransformer](https://huggingface.co/docs/transformers/model_doc/speech_to_text)** (来自 Facebook), 伴随论文 [fairseq S2T: Fast Speech-to-Text Modeling with fairseq](https://arxiv.org/abs/2010.05171) 由 Changhan Wang, Yun Tang, Xutai Ma, Anne Wu, Dmytro Okhonko, Juan Pino 发布。 1. **[SpeechToTextTransformer2](https://huggingface.co/docs/transformers/model_doc/speech_to_text_2)** (来自 Facebook) 伴随论文 [Large-Scale Self- and Semi-Supervised Learning for Speech Translation](https://arxiv.org/abs/2104.06678) 由 Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli, Alexis Conneau 发布。 1. **[Splinter](https://huggingface.co/docs/transformers/model_doc/splinter)** (来自 Tel Aviv University) 伴随论文 [Few-Shot Question Answering by Pretraining Span Selection](https://arxiv.org/abs/2101.00438) 由 Ori Ram, Yuval Kirstain, Jonathan Berant, Amir Globerson, Omer Levy 发布。 @@ -395,7 +395,7 @@ conda install -c huggingface transformers 1. **[Trajectory Transformer](https://huggingface.co/docs/transformers/model_doc/trajectory_transformers)** (from the University of California at Berkeley) released with the paper [Offline Reinforcement Learning as One Big Sequence Modeling Problem](https://arxiv.org/abs/2106.02039) by Michael Janner, Qiyang Li, Sergey Levine 1. **[Transformer-XL](https://huggingface.co/docs/transformers/model_doc/transfo-xl)** (来自 Google/CMU) 伴随论文 [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) 由 Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov 发布。 1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (来自 Microsoft) 伴随论文 [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) 由 Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei 发布。 -1. **[TVLT](https://huggingface.co/docs/transformers/main/model_doc/tvlt)** (来自 UNC Chapel Hill) 伴随论文 [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) 由 Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal 发布。 +1. **[TVLT](https://huggingface.co/docs/transformers/model_doc/tvlt)** (来自 UNC Chapel Hill) 伴随论文 [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) 由 Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal 发布。 1. **[UL2](https://huggingface.co/docs/transformers/model_doc/ul2)** (from Google Research) released with the paper [Unifying Language Learning Paradigms](https://arxiv.org/abs/2205.05131v1) by Yi Tay, Mostafa Dehghani, Vinh Q. Tran, Xavier Garcia, Dara Bahri, Tal Schuster, Huaixiu Steven Zheng, Neil Houlsby, Donald Metzler 1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (来自 Microsoft Research) 伴随论文 [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) 由 Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang 发布。 1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (来自 Microsoft Research) 伴随论文 [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://arxiv.org/abs/2110.05752) 由 Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu 发布。 @@ -414,7 +414,7 @@ conda install -c huggingface transformers 1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (来自 OpenAI) 伴随论文 [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) 由 Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever 发布。 1. **[X-CLIP](https://huggingface.co/docs/transformers/model_doc/xclip)** (来自 Microsoft Research) 伴随论文 [Expanding Language-Image Pretrained Models for General Video Recognition](https://arxiv.org/abs/2208.02816) 由 Bolin Ni, Houwen Peng, Minghao Chen, Songyang Zhang, Gaofeng Meng, Jianlong Fu, Shiming Xiang, Haibin Ling 发布。 -1. **[X-MOD](https://huggingface.co/docs/transformers/main/model_doc/xmod)** (来自 Meta AI) 伴随论文 [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) 由 Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe 发布。 +1. **[X-MOD](https://huggingface.co/docs/transformers/model_doc/xmod)** (来自 Meta AI) 伴随论文 [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) 由 Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe 发布。 1. **[XGLM](https://huggingface.co/docs/transformers/model_doc/xglm)** (From Facebook AI) released with the paper [Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668) by Xi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (来自 Facebook) 伴随论文 [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) 由 Guillaume Lample and Alexis Conneau 发布。 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlm-prophetnet)** (来自 Microsoft Research) 伴随论文 [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) 由 Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index 7fa3f4a6c3a0..f5b087f412e1 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -249,7 +249,7 @@ conda install -c huggingface transformers 🤗 Transformers 目前支援以下的架構（模型概覽請參閱[這裡](https://huggingface.co/docs/transformers/model_summary)）： 1. **[ALBERT](https://huggingface.co/docs/transformers/model_doc/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. -1. **[ALIGN](https://huggingface.co/docs/transformers/main/model_doc/align)** (from Google Research) released with the paper [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. +1. **[ALIGN](https://huggingface.co/docs/transformers/model_doc/align)** (from Google Research) released with the paper [Scaling Up Visual and Vision-Language Representation Learning With Noisy Text Supervision](https://arxiv.org/abs/2102.05918) by Chao Jia, Yinfei Yang, Ye Xia, Yi-Ting Chen, Zarana Parekh, Hieu Pham, Quoc V. Le, Yunhsuan Sung, Zhen Li, Tom Duerig. 1. **[AltCLIP](https://huggingface.co/docs/transformers/model_doc/altclip)** (from BAAI) released with the paper [AltCLIP: Altering the Language Encoder in CLIP for Extended Language Capabilities](https://arxiv.org/abs/2211.06679) by Chen, Zhongzhi and Liu, Guang and Zhang, Bo-Wen and Ye, Fulong and Yang, Qinghong and Wu, Ledell. 1. **[Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer)** (from MIT) released with the paper [AST: Audio Spectrogram Transformer](https://arxiv.org/abs/2104.01778) by Yuan Gong, Yu-An Chung, James Glass. 1. **[BART](https://huggingface.co/docs/transformers/model_doc/bart)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/pdf/1910.13461.pdf) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer. @@ -266,22 +266,22 @@ conda install -c huggingface transformers 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BLIP](https://huggingface.co/docs/transformers/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. -1. **[BLIP-2](https://huggingface.co/docs/transformers/main/model_doc/blip-2)** (from Salesforce) released with the paper [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) by Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. +1. **[BLIP-2](https://huggingface.co/docs/transformers/model_doc/blip-2)** (from Salesforce) released with the paper [BLIP-2: Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models](https://arxiv.org/abs/2301.12597) by Junnan Li, Dongxu Li, Silvio Savarese, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. -1. **[BridgeTower](https://huggingface.co/docs/transformers/main/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. +1. **[BridgeTower](https://huggingface.co/docs/transformers/model_doc/bridgetower)** (from Harbin Institute of Technology/Microsoft Research Asia/Intel Labs) released with the paper [BridgeTower: Building Bridges Between Encoders in Vision-Language Representation Learning](https://arxiv.org/abs/2206.08657) by Xiao Xu, Chenfei Wu, Shachar Rosenman, Vasudev Lal, Wanxiang Che, Nan Duan. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. 1. **[CamemBERT](https://huggingface.co/docs/transformers/model_doc/camembert)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 1. **[CANINE](https://huggingface.co/docs/transformers/model_doc/canine)** (from Google Research) released with the paper [CANINE: Pre-training an Efficient Tokenization-Free Encoder for Language Representation](https://arxiv.org/abs/2103.06874) by Jonathan H. Clark, Dan Garrette, Iulia Turc, John Wieting. 1. **[Chinese-CLIP](https://huggingface.co/docs/transformers/model_doc/chinese_clip)** (from OFA-Sys) released with the paper [Chinese CLIP: Contrastive Vision-Language Pretraining in Chinese](https://arxiv.org/abs/2211.01335) by An Yang, Junshu Pan, Junyang Lin, Rui Men, Yichang Zhang, Jingren Zhou, Chang Zhou. -1. **[CLAP](https://huggingface.co/docs/transformers/main/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. +1. **[CLAP](https://huggingface.co/docs/transformers/model_doc/clap)** (from LAION-AI) released with the paper [Large-scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation]https://arxiv.org/abs/2211.06687) by Yusong Wu, Ke Chen, Tianyu Zhang, Yuchen Hui, Taylor Berg-Kirkpatrick, Shlomo Dubnov. 1. **[CLIP](https://huggingface.co/docs/transformers/model_doc/clip)** (from OpenAI) released with the paper [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) by Alec Radford, Jong Wook Kim, Chris Hallacy, Aditya Ramesh, Gabriel Goh, Sandhini Agarwal, Girish Sastry, Amanda Askell, Pamela Mishkin, Jack Clark, Gretchen Krueger, Ilya Sutskever. 1. **[CLIPSeg](https://huggingface.co/docs/transformers/model_doc/clipseg)** (from University of Göttingen) released with the paper [Image Segmentation Using Text and Image Prompts](https://arxiv.org/abs/2112.10003) by Timo Lüddecke and Alexander Ecker. 1. **[CodeGen](https://huggingface.co/docs/transformers/model_doc/codegen)** (from Salesforce) released with the paper [A Conversational Paradigm for Program Synthesis](https://arxiv.org/abs/2203.13474) by Erik Nijkamp, Bo Pang, Hiroaki Hayashi, Lifu Tu, Huan Wang, Yingbo Zhou, Silvio Savarese, Caiming Xiong. 1. **[Conditional DETR](https://huggingface.co/docs/transformers/model_doc/conditional_detr)** (from Microsoft Research Asia) released with the paper [Conditional DETR for Fast Training Convergence](https://arxiv.org/abs/2108.06152) by Depu Meng, Xiaokang Chen, Zejia Fan, Gang Zeng, Houqiang Li, Yuhui Yuan, Lei Sun, Jingdong Wang. 1. **[ConvBERT](https://huggingface.co/docs/transformers/model_doc/convbert)** (from YituTech) released with the paper [ConvBERT: Improving BERT with Span-based Dynamic Convolution](https://arxiv.org/abs/2008.02496) by Zihang Jiang, Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan. 1. **[ConvNeXT](https://huggingface.co/docs/transformers/model_doc/convnext)** (from Facebook AI) released with the paper [A ConvNet for the 2020s](https://arxiv.org/abs/2201.03545) by Zhuang Liu, Hanzi Mao, Chao-Yuan Wu, Christoph Feichtenhofer, Trevor Darrell, Saining Xie. -1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/main/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. +1. **[ConvNeXTV2](https://huggingface.co/docs/transformers/model_doc/convnextv2)** (from Facebook AI) released with the paper [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](https://arxiv.org/abs/2301.00808) by Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon, Saining Xie. 1. **[CPM](https://huggingface.co/docs/transformers/model_doc/cpm)** (from Tsinghua University) released with the paper [CPM: A Large-scale Generative Chinese Pre-trained Language Model](https://arxiv.org/abs/2012.00413) by Zhengyan Zhang, Xu Han, Hao Zhou, Pei Ke, Yuxian Gu, Deming Ye, Yujia Qin, Yusheng Su, Haozhe Ji, Jian Guan, Fanchao Qi, Xiaozhi Wang, Yanan Zheng, Guoyang Zeng, Huanqi Cao, Shengqi Chen, Daixuan Li, Zhenbo Sun, Zhiyuan Liu, Minlie Huang, Wentao Han, Jie Tang, Juanzi Li, Xiaoyan Zhu, Maosong Sun. 1. **[CTRL](https://huggingface.co/docs/transformers/model_doc/ctrl)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 1. **[CvT](https://huggingface.co/docs/transformers/model_doc/cvt)** (from Microsoft) released with the paper [CvT: Introducing Convolutions to Vision Transformers](https://arxiv.org/abs/2103.15808) by Haiping Wu, Bin Xiao, Noel Codella, Mengchen Liu, Xiyang Dai, Lu Yuan, Lei Zhang. @@ -291,7 +291,7 @@ conda install -c huggingface transformers 1. **[Decision Transformer](https://huggingface.co/docs/transformers/model_doc/decision_transformer)** (from Berkeley/Facebook/Google) released with the paper [Decision Transformer: Reinforcement Learning via Sequence Modeling](https://arxiv.org/abs/2106.01345) by Lili Chen, Kevin Lu, Aravind Rajeswaran, Kimin Lee, Aditya Grover, Michael Laskin, Pieter Abbeel, Aravind Srinivas, Igor Mordatch. 1. **[Deformable DETR](https://huggingface.co/docs/transformers/model_doc/deformable_detr)** (from SenseTime Research) released with the paper [Deformable DETR: Deformable Transformers for End-to-End Object Detection](https://arxiv.org/abs/2010.04159) by Xizhou Zhu, Weijie Su, Lewei Lu, Bin Li, Xiaogang Wang, Jifeng Dai. 1. **[DeiT](https://huggingface.co/docs/transformers/model_doc/deit)** (from Facebook) released with the paper [Training data-efficient image transformers & distillation through attention](https://arxiv.org/abs/2012.12877) by Hugo Touvron, Matthieu Cord, Matthijs Douze, Francisco Massa, Alexandre Sablayrolles, Hervé Jégou. -1. **[DETA](https://huggingface.co/docs/transformers/main/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. +1. **[DETA](https://huggingface.co/docs/transformers/model_doc/deta)** (from The University of Texas at Austin) released with the paper [NMS Strikes Back](https://arxiv.org/abs/2212.06137) by Jeffrey Ouyang-Zhang, Jang Hyun Cho, Xingyi Zhou, Philipp Krähenbühl. 1. **[DETR](https://huggingface.co/docs/transformers/model_doc/detr)** (from Facebook) released with the paper [End-to-End Object Detection with Transformers](https://arxiv.org/abs/2005.12872) by Nicolas Carion, Francisco Massa, Gabriel Synnaeve, Nicolas Usunier, Alexander Kirillov, Sergey Zagoruyko. 1. **[DialoGPT](https://huggingface.co/docs/transformers/model_doc/dialogpt)** (from Microsoft Research) released with the paper [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan. 1. **[DiNAT](https://huggingface.co/docs/transformers/model_doc/dinat)** (from SHI Labs) released with the paper [Dilated Neighborhood Attention Transformer](https://arxiv.org/abs/2209.15001) by Ali Hassani and Humphrey Shi. @@ -301,11 +301,11 @@ conda install -c huggingface transformers 1. **[DPR](https://huggingface.co/docs/transformers/model_doc/dpr)** (from Facebook) released with the paper [Dense Passage Retrieval for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) by Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih. 1. **[DPT](https://huggingface.co/docs/transformers/master/model_doc/dpt)** (from Intel Labs) released with the paper [Vision Transformers for Dense Prediction](https://arxiv.org/abs/2103.13413) by René Ranftl, Alexey Bochkovskiy, Vladlen Koltun. 1. **[EfficientFormer](https://huggingface.co/docs/transformers/model_doc/efficientformer)** (from Snap Research) released with the paper [EfficientFormer: Vision Transformers at MobileNetSpeed](https://arxiv.org/abs/2206.01191) by Yanyu Li, Geng Yuan, Yang Wen, Ju Hu, Georgios Evangelidis, Sergey Tulyakov, Yanzhi Wang, Jian Ren. -1. **[EfficientNet](https://huggingface.co/docs/transformers/main/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. +1. **[EfficientNet](https://huggingface.co/docs/transformers/model_doc/efficientnet)** (from Google Brain) released with the paper [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) by Mingxing Tan, Quoc V. Le. 1. **[ELECTRA](https://huggingface.co/docs/transformers/model_doc/electra)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning. 1. **[EncoderDecoder](https://huggingface.co/docs/transformers/model_doc/encoder-decoder)** (from Google Research) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. 1. **[ERNIE](https://huggingface.co/docs/transformers/model_doc/ernie)** (from Baidu) released with the paper [ERNIE: Enhanced Representation through Knowledge Integration](https://arxiv.org/abs/1904.09223) by Yu Sun, Shuohuan Wang, Yukun Li, Shikun Feng, Xuyi Chen, Han Zhang, Xin Tian, Danxiang Zhu, Hao Tian, Hua Wu. -1. **[ErnieM](https://huggingface.co/docs/transformers/main/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. +1. **[ErnieM](https://huggingface.co/docs/transformers/model_doc/ernie_m)** (from Baidu) released with the paper [ERNIE-M: Enhanced Multilingual Representation by Aligning Cross-lingual Semantics with Monolingual Corpora](https://arxiv.org/abs/2012.15674) by Xuan Ouyang, Shuohuan Wang, Chao Pang, Yu Sun, Hao Tian, Hua Wu, Haifeng Wang. 1. **[ESM](https://huggingface.co/docs/transformers/model_doc/esm)** (from Meta AI) are transformer protein language models. **ESM-1b** was released with the paper [Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences](https://www.pnas.org/content/118/15/e2016239118) by Alexander Rives, Joshua Meier, Tom Sercu, Siddharth Goyal, Zeming Lin, Jason Liu, Demi Guo, Myle Ott, C. Lawrence Zitnick, Jerry Ma, and Rob Fergus. **ESM-1v** was released with the paper [Language models enable zero-shot prediction of the effects of mutations on protein function](https://doi.org/10.1101/2021.07.09.450648) by Joshua Meier, Roshan Rao, Robert Verkuil, Jason Liu, Tom Sercu and Alexander Rives. **ESM-2** was released with the paper [Language models of protein sequences at the scale of evolution enable accurate structure prediction](https://doi.org/10.1101/2022.07.20.500902) by Zeming Lin, Halil Akin, Roshan Rao, Brian Hie, Zhongkai Zhu, Wenting Lu, Allan dos Santos Costa, Maryam Fazel-Zarandi, Tom Sercu, Sal Candido, Alexander Rives. 1. **[FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-t5-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei 1. **[FLAN-UL2](https://huggingface.co/docs/transformers/model_doc/flan-ul2)** (from Google AI) released in the repository [google-research/t5x](https://github.com/google-research/t5x/blob/main/docs/models.md#flan-ul2-checkpoints) by Hyung Won Chung, Le Hou, Shayne Longpre, Barret Zoph, Yi Tay, William Fedus, Eric Li, Xuezhi Wang, Mostafa Dehghani, Siddhartha Brahma, Albert Webson, Shixiang Shane Gu, Zhuyun Dai, Mirac Suzgun, Xinyun Chen, Aakanksha Chowdhery, Sharan Narang, Gaurav Mishra, Adams Yu, Vincent Zhao, Yanping Huang, Andrew Dai, Hongkun Yu, Slav Petrov, Ed H. Chi, Jeff Dean, Jacob Devlin, Adam Roberts, Denny Zhou, Quoc V. Le, and Jason Wei @@ -322,13 +322,13 @@ conda install -c huggingface transformers 1. **[GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. 1. **[GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj)** (from EleutherAI) released with the paper [kingoflolz/mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/) by Ben Wang and Aran Komatsuzaki. 1. **[GPT-Sw3](https://huggingface.co/docs/transformers/model_doc/gpt-sw3)** (from AI-Sweden) released with the paper [Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish](http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf) by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman, Fredrik Carlsson, Magnus Sahlgren. -1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/main/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by 坂本俊之(tanreinama). +1. **[GPTSAN-japanese](https://huggingface.co/docs/transformers/model_doc/gptsan-japanese)** released in the repository [tanreinama/GPTSAN](https://github.com/tanreinama/GPTSAN/blob/main/report/model.md) by 坂本俊之(tanreinama). 1. **[Graphormer](https://huggingface.co/docs/transformers/model_doc/graphormer)** (from Microsoft) released with the paper [Do Transformers Really Perform Bad for Graph Representation?](https://arxiv.org/abs/2106.05234) by Chengxuan Ying, Tianle Cai, Shengjie Luo, Shuxin Zheng, Guolin Ke, Di He, Yanming Shen, Tie-Yan Liu. 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://arxiv.org/abs/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang. 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://arxiv.org/abs/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed. 1. **[I-BERT](https://huggingface.co/docs/transformers/model_doc/ibert)** (from Berkeley) released with the paper [I-BERT: Integer-only BERT Quantization](https://arxiv.org/abs/2101.01321) by Sehoon Kim, Amir Gholami, Zhewei Yao, Michael W. Mahoney, Kurt Keutzer. 1. **[ImageGPT](https://huggingface.co/docs/transformers/model_doc/imagegpt)** (from OpenAI) released with the paper [Generative Pretraining from Pixels](https://openai.com/blog/image-gpt/) by Mark Chen, Alec Radford, Rewon Child, Jeffrey Wu, Heewoo Jun, David Luan, Ilya Sutskever. -1. **[Informer](https://huggingface.co/docs/transformers/main/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. +1. **[Informer](https://huggingface.co/docs/transformers/model_doc/informer)** (from Beihang University, UC Berkeley, Rutgers University, SEDD Company) released with the paper [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting](https://arxiv.org/abs/2012.07436) by Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. 1. **[Jukebox](https://huggingface.co/docs/transformers/model_doc/jukebox)** (from OpenAI) released with the paper [Jukebox: A Generative Model for Music](https://arxiv.org/pdf/2005.00341.pdf) by Prafulla Dhariwal, Heewoo Jun, Christine Payne, Jong Wook Kim, Alec Radford, Ilya Sutskever. 1. **[LayoutLM](https://huggingface.co/docs/transformers/model_doc/layoutlm)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou. 1. **[LayoutLMv2](https://huggingface.co/docs/transformers/model_doc/layoutlmv2)** (from Microsoft Research Asia) released with the paper [LayoutLMv2: Multi-modal Pre-training for Visually-Rich Document Understanding](https://arxiv.org/abs/2012.14740) by Yang Xu, Yiheng Xu, Tengchao Lv, Lei Cui, Furu Wei, Guoxin Wang, Yijuan Lu, Dinei Florencio, Cha Zhang, Wanxiang Che, Min Zhang, Lidong Zhou. @@ -351,7 +351,7 @@ conda install -c huggingface transformers 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. -1. **[MGP-STR](https://huggingface.co/docs/transformers/main/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. +1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. 1. **[mLUKE](https://huggingface.co/docs/transformers/model_doc/mluke)** (from Studio Ousia) released with the paper [mLUKE: The Power of Entity Representations in Multilingual Pretrained Language Models](https://arxiv.org/abs/2110.08151) by Ryokan Ri, Ikuya Yamada, and Yoshimasa Tsuruoka. 1. **[MobileBERT](https://huggingface.co/docs/transformers/model_doc/mobilebert)** (from CMU/Google Brain) released with the paper [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. 1. **[MobileNetV1](https://huggingface.co/docs/transformers/model_doc/mobilenet_v1)** (from Google Inc.) released with the paper [MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications](https://arxiv.org/abs/1704.04861) by Andrew G. Howard, Menglong Zhu, Bo Chen, Dmitry Kalenichenko, Weijun Wang, Tobias Weyand, Marco Andreetto, Hartwig Adam. @@ -388,7 +388,7 @@ conda install -c huggingface transformers 1. **[SegFormer](https://huggingface.co/docs/transformers/model_doc/segformer)** (from NVIDIA) released with the paper [SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers](https://arxiv.org/abs/2105.15203) by Enze Xie, Wenhai Wang, Zhiding Yu, Anima Anandkumar, Jose M. Alvarez, Ping Luo. 1. **[SEW](https://huggingface.co/docs/transformers/model_doc/sew)** (from ASAPP) released with the paper [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi. 1. **[SEW-D](https://huggingface.co/docs/transformers/model_doc/sew_d)** (from ASAPP) released with the paper [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi. -1. **[SpeechT5](https://huggingface.co/docs/transformers/main/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. +1. **[SpeechT5](https://huggingface.co/docs/transformers/model_doc/speecht5)** (from Microsoft Research) released with the paper [SpeechT5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing](https://arxiv.org/abs/2110.07205) by Junyi Ao, Rui Wang, Long Zhou, Chengyi Wang, Shuo Ren, Yu Wu, Shujie Liu, Tom Ko, Qing Li, Yu Zhang, Zhihua Wei, Yao Qian, Jinyu Li, Furu Wei. 1. **[SpeechToTextTransformer](https://huggingface.co/docs/transformers/model_doc/speech_to_text)** (from Facebook), released together with the paper [fairseq S2T: Fast Speech-to-Text Modeling with fairseq](https://arxiv.org/abs/2010.05171) by Changhan Wang, Yun Tang, Xutai Ma, Anne Wu, Dmytro Okhonko, Juan Pino. 1. **[SpeechToTextTransformer2](https://huggingface.co/docs/transformers/model_doc/speech_to_text_2)** (from Facebook) released with the paper [Large-Scale Self- and Semi-Supervised Learning for Speech Translation](https://arxiv.org/abs/2104.06678) by Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli, Alexis Conneau. 1. **[Splinter](https://huggingface.co/docs/transformers/model_doc/splinter)** (from Tel Aviv University) released with the paper [Few-Shot Question Answering by Pretraining Span Selection](https://arxiv.org/abs/2101.00438) by Ori Ram, Yuval Kirstain, Jonathan Berant, Amir Globerson, Omer Levy. @@ -407,7 +407,7 @@ conda install -c huggingface transformers 1. **[Trajectory Transformer](https://huggingface.co/docs/transformers/model_doc/trajectory_transformers)** (from the University of California at Berkeley) released with the paper [Offline Reinforcement Learning as One Big Sequence Modeling Problem](https://arxiv.org/abs/2106.02039) by Michael Janner, Qiyang Li, Sergey Levine 1. **[Transformer-XL](https://huggingface.co/docs/transformers/model_doc/transfo-xl)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. 1. **[TrOCR](https://huggingface.co/docs/transformers/model_doc/trocr)** (from Microsoft) released with the paper [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang, Zhoujun Li, Furu Wei. -1. **[TVLT](https://huggingface.co/docs/transformers/main/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. +1. **[TVLT](https://huggingface.co/docs/transformers/model_doc/tvlt)** (from UNC Chapel Hill) released with the paper [TVLT: Textless Vision-Language Transformer](https://arxiv.org/abs/2209.14156) by Zineng Tang, Jaemin Cho, Yixin Nie, Mohit Bansal. 1. **[UL2](https://huggingface.co/docs/transformers/model_doc/ul2)** (from Google Research) released with the paper [Unifying Language Learning Paradigms](https://arxiv.org/abs/2205.05131v1) by Yi Tay, Mostafa Dehghani, Vinh Q. Tran, Xavier Garcia, Dara Bahri, Tal Schuster, Huaixiu Steven Zheng, Neil Houlsby, Donald Metzler 1. **[UniSpeech](https://huggingface.co/docs/transformers/model_doc/unispeech)** (from Microsoft Research) released with the paper [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang. 1. **[UniSpeechSat](https://huggingface.co/docs/transformers/model_doc/unispeech-sat)** (from Microsoft Research) released with the paper [UNISPEECH-SAT: UNIVERSAL SPEECH REPRESENTATION LEARNING WITH SPEAKER AWARE PRE-TRAINING](https://arxiv.org/abs/2110.05752) by Sanyuan Chen, Yu Wu, Chengyi Wang, Zhengyang Chen, Zhuo Chen, Shujie Liu, Jian Wu, Yao Qian, Furu Wei, Jinyu Li, Xiangzhan Yu. @@ -426,7 +426,7 @@ conda install -c huggingface transformers 1. **[WavLM](https://huggingface.co/docs/transformers/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[Whisper](https://huggingface.co/docs/transformers/model_doc/whisper)** (from OpenAI) released with the paper [Robust Speech Recognition via Large-Scale Weak Supervision](https://cdn.openai.com/papers/whisper.pdf) by Alec Radford, Jong Wook Kim, Tao Xu, Greg Brockman, Christine McLeavey, Ilya Sutskever. 1. **[X-CLIP](https://huggingface.co/docs/transformers/model_doc/xclip)** (from Microsoft Research) released with the paper [Expanding Language-Image Pretrained Models for General Video Recognition](https://arxiv.org/abs/2208.02816) by Bolin Ni, Houwen Peng, Minghao Chen, Songyang Zhang, Gaofeng Meng, Jianlong Fu, Shiming Xiang, Haibin Ling. -1. **[X-MOD](https://huggingface.co/docs/transformers/main/model_doc/xmod)** (from Meta AI) released with the paper [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) by Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. +1. **[X-MOD](https://huggingface.co/docs/transformers/model_doc/xmod)** (from Meta AI) released with the paper [Lifting the Curse of Multilinguality by Pre-training Modular Transformers](http://dx.doi.org/10.18653/v1/2022.naacl-main.255) by Jonas Pfeiffer, Naman Goyal, Xi Lin, Xian Li, James Cross, Sebastian Riedel, Mikel Artetxe. 1. **[XGLM](https://huggingface.co/docs/transformers/model_doc/xglm)** (From Facebook AI) released with the paper [Few-shot Learning with Multilingual Language Models](https://arxiv.org/abs/2112.10668) by Xi Victoria Lin, Todor Mihaylov, Mikel Artetxe, Tianlu Wang, Shuohui Chen, Daniel Simig, Myle Ott, Naman Goyal, Shruti Bhosale, Jingfei Du, Ramakanth Pasunuru, Sam Shleifer, Punit Singh Koura, Vishrav Chaudhary, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Zornitsa Kozareva, Mona Diab, Veselin Stoyanov, Xian Li. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlm-prophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. diff --git a/examples/flax/question-answering/run_qa.py b/examples/flax/question-answering/run_qa.py index 7933c3bd3e58..86cd87f233ce 100644 --- a/examples/flax/question-answering/run_qa.py +++ b/examples/flax/question-answering/run_qa.py @@ -61,7 +61,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") Array = Any Dataset = datasets.arrow_dataset.Dataset diff --git a/examples/flax/text-classification/run_flax_glue.py b/examples/flax/text-classification/run_flax_glue.py index 4fd12404d436..4f1019a5c4cd 100755 --- a/examples/flax/text-classification/run_flax_glue.py +++ b/examples/flax/text-classification/run_flax_glue.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") Array = Any Dataset = datasets.arrow_dataset.Dataset diff --git a/examples/flax/token-classification/run_flax_ner.py b/examples/flax/token-classification/run_flax_ner.py index d17676528915..f167cad2290d 100644 --- a/examples/flax/token-classification/run_flax_ner.py +++ b/examples/flax/token-classification/run_flax_ner.py @@ -55,7 +55,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/token-classification/requirements.txt") diff --git a/examples/pytorch/audio-classification/run_audio_classification.py b/examples/pytorch/audio-classification/run_audio_classification.py index 2231e96dc4e5..2333b08b5830 100644 --- a/examples/pytorch/audio-classification/run_audio_classification.py +++ b/examples/pytorch/audio-classification/run_audio_classification.py @@ -45,7 +45,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.14.0", "To fix: pip install -r examples/pytorch/audio-classification/requirements.txt") diff --git a/examples/pytorch/contrastive-image-text/run_clip.py b/examples/pytorch/contrastive-image-text/run_clip.py index 8db1e67f9479..749582b782e5 100644 --- a/examples/pytorch/contrastive-image-text/run_clip.py +++ b/examples/pytorch/contrastive-image-text/run_clip.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/contrastive-image-text/requirements.txt") diff --git a/examples/pytorch/image-classification/run_image_classification.py b/examples/pytorch/image-classification/run_image_classification.py index 2fb7f49a85aa..e419028e70e0 100644 --- a/examples/pytorch/image-classification/run_image_classification.py +++ b/examples/pytorch/image-classification/run_image_classification.py @@ -55,7 +55,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/image-classification/requirements.txt") diff --git a/examples/pytorch/image-classification/run_image_classification_no_trainer.py b/examples/pytorch/image-classification/run_image_classification_no_trainer.py index 3ba79d630e76..33e96a1656fa 100644 --- a/examples/pytorch/image-classification/run_image_classification_no_trainer.py +++ b/examples/pytorch/image-classification/run_image_classification_no_trainer.py @@ -47,7 +47,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) diff --git a/examples/pytorch/image-pretraining/run_mae.py b/examples/pytorch/image-pretraining/run_mae.py index 231431bf5d97..c1f6ba26a427 100644 --- a/examples/pytorch/image-pretraining/run_mae.py +++ b/examples/pytorch/image-pretraining/run_mae.py @@ -43,7 +43,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/image-pretraining/requirements.txt") diff --git a/examples/pytorch/image-pretraining/run_mim.py b/examples/pytorch/image-pretraining/run_mim.py index 8c744bb41a5b..0997858901eb 100644 --- a/examples/pytorch/image-pretraining/run_mim.py +++ b/examples/pytorch/image-pretraining/run_mim.py @@ -48,7 +48,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/image-pretraining/requirements.txt") diff --git a/examples/pytorch/language-modeling/run_clm.py b/examples/pytorch/language-modeling/run_clm.py index 6f8854226845..9b83c8e2ee57 100755 --- a/examples/pytorch/language-modeling/run_clm.py +++ b/examples/pytorch/language-modeling/run_clm.py @@ -55,7 +55,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") diff --git a/examples/pytorch/language-modeling/run_clm_no_trainer.py b/examples/pytorch/language-modeling/run_clm_no_trainer.py index 998ca60b2700..e43ff3855536 100755 --- a/examples/pytorch/language-modeling/run_clm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_clm_no_trainer.py @@ -57,7 +57,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) diff --git a/examples/pytorch/language-modeling/run_mlm.py b/examples/pytorch/language-modeling/run_mlm.py index 7cccffe9bd94..a8d3d8f29aef 100755 --- a/examples/pytorch/language-modeling/run_mlm.py +++ b/examples/pytorch/language-modeling/run_mlm.py @@ -53,7 +53,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") diff --git a/examples/pytorch/language-modeling/run_mlm_no_trainer.py b/examples/pytorch/language-modeling/run_mlm_no_trainer.py index ee469e48890e..78cf3134997d 100755 --- a/examples/pytorch/language-modeling/run_mlm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_mlm_no_trainer.py @@ -57,7 +57,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") diff --git a/examples/pytorch/language-modeling/run_plm.py b/examples/pytorch/language-modeling/run_plm.py index 562d43802922..582fe5215bb5 100755 --- a/examples/pytorch/language-modeling/run_plm.py +++ b/examples/pytorch/language-modeling/run_plm.py @@ -47,7 +47,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") diff --git a/examples/pytorch/multiple-choice/run_swag.py b/examples/pytorch/multiple-choice/run_swag.py index 1191595944a5..934b77d503e1 100755 --- a/examples/pytorch/multiple-choice/run_swag.py +++ b/examples/pytorch/multiple-choice/run_swag.py @@ -47,7 +47,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = logging.getLogger(__name__) diff --git a/examples/pytorch/multiple-choice/run_swag_no_trainer.py b/examples/pytorch/multiple-choice/run_swag_no_trainer.py index b0bcc567551c..f4a358719a57 100755 --- a/examples/pytorch/multiple-choice/run_swag_no_trainer.py +++ b/examples/pytorch/multiple-choice/run_swag_no_trainer.py @@ -56,7 +56,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) # You should update this to your particular problem to have better documentation of `model_type` diff --git a/examples/pytorch/question-answering/run_qa.py b/examples/pytorch/question-answering/run_qa.py index 8d20eab44d90..e6a1d421b9a7 100755 --- a/examples/pytorch/question-answering/run_qa.py +++ b/examples/pytorch/question-answering/run_qa.py @@ -49,7 +49,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/question-answering/requirements.txt") diff --git a/examples/pytorch/question-answering/run_qa_beam_search.py b/examples/pytorch/question-answering/run_qa_beam_search.py index d265b093cf7a..5321d184af28 100755 --- a/examples/pytorch/question-answering/run_qa_beam_search.py +++ b/examples/pytorch/question-answering/run_qa_beam_search.py @@ -48,7 +48,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/question-answering/requirements.txt") diff --git a/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py b/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py index 9372de3298f2..fa59a241422c 100644 --- a/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py @@ -56,7 +56,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/question-answering/requirements.txt") diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py index 6cbea37151da..21db900c65ba 100755 --- a/examples/pytorch/question-answering/run_qa_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_no_trainer.py @@ -57,7 +57,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/question-answering/requirements.txt") diff --git a/examples/pytorch/question-answering/run_seq2seq_qa.py b/examples/pytorch/question-answering/run_seq2seq_qa.py index adf3c4269e90..00a30685e538 100644 --- a/examples/pytorch/question-answering/run_seq2seq_qa.py +++ b/examples/pytorch/question-answering/run_seq2seq_qa.py @@ -45,7 +45,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/question-answering/requirements.txt") diff --git a/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py b/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py index a6aeb75c1b83..b50375535578 100644 --- a/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py +++ b/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=2.0.0", "To fix: pip install -r examples/pytorch/semantic-segmentation/requirements.txt") diff --git a/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py b/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py index 702adb0151ab..791291dd356e 100644 --- a/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py +++ b/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py @@ -50,7 +50,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py index f600c03f232b..a6c71e5e7035 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py @@ -51,7 +51,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.18.0", "To fix: pip install -r examples/pytorch/speech-recognition/requirements.txt") diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py index f27f70016288..86213c47f576 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py @@ -48,7 +48,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.18.0", "To fix: pip install -r examples/pytorch/speech-recognition/requirements.txt") diff --git a/examples/pytorch/summarization/run_summarization.py b/examples/pytorch/summarization/run_summarization.py index 2645f677f1c6..e5566f936328 100755 --- a/examples/pytorch/summarization/run_summarization.py +++ b/examples/pytorch/summarization/run_summarization.py @@ -52,7 +52,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/summarization/requirements.txt") diff --git a/examples/pytorch/summarization/run_summarization_no_trainer.py b/examples/pytorch/summarization/run_summarization_no_trainer.py index b16a3fd06900..e0c4b313ad76 100644 --- a/examples/pytorch/summarization/run_summarization_no_trainer.py +++ b/examples/pytorch/summarization/run_summarization_no_trainer.py @@ -56,7 +56,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/summarization/requirements.txt") diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py index ebfaae0cbe5f..f05967f8baf8 100755 --- a/examples/pytorch/text-classification/run_glue.py +++ b/examples/pytorch/text-classification/run_glue.py @@ -48,7 +48,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt") diff --git a/examples/pytorch/text-classification/run_glue_no_trainer.py b/examples/pytorch/text-classification/run_glue_no_trainer.py index ee7438071f23..71c3686f6764 100644 --- a/examples/pytorch/text-classification/run_glue_no_trainer.py +++ b/examples/pytorch/text-classification/run_glue_no_trainer.py @@ -48,7 +48,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) diff --git a/examples/pytorch/text-classification/run_xnli.py b/examples/pytorch/text-classification/run_xnli.py index de8d7b9ce540..d26f59d0e314 100755 --- a/examples/pytorch/text-classification/run_xnli.py +++ b/examples/pytorch/text-classification/run_xnli.py @@ -48,7 +48,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt") diff --git a/examples/pytorch/token-classification/run_ner.py b/examples/pytorch/token-classification/run_ner.py index 8eccb54759b4..cf9b89662281 100755 --- a/examples/pytorch/token-classification/run_ner.py +++ b/examples/pytorch/token-classification/run_ner.py @@ -49,7 +49,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/token-classification/requirements.txt") diff --git a/examples/pytorch/token-classification/run_ner_no_trainer.py b/examples/pytorch/token-classification/run_ner_no_trainer.py index 0c6fa85b6b82..1007ae2ca6ec 100755 --- a/examples/pytorch/token-classification/run_ner_no_trainer.py +++ b/examples/pytorch/token-classification/run_ner_no_trainer.py @@ -55,7 +55,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/token-classification/requirements.txt") diff --git a/examples/pytorch/translation/run_translation.py b/examples/pytorch/translation/run_translation.py index 4a47a4606425..19d7c587bfd7 100755 --- a/examples/pytorch/translation/run_translation.py +++ b/examples/pytorch/translation/run_translation.py @@ -52,7 +52,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/translation/requirements.txt") diff --git a/examples/pytorch/translation/run_translation_no_trainer.py b/examples/pytorch/translation/run_translation_no_trainer.py index a853d531edb8..8c29db4d6db2 100644 --- a/examples/pytorch/translation/run_translation_no_trainer.py +++ b/examples/pytorch/translation/run_translation_no_trainer.py @@ -57,7 +57,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = get_logger(__name__) require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/translation/requirements.txt") diff --git a/examples/tensorflow/contrastive-image-text/run_clip.py b/examples/tensorflow/contrastive-image-text/run_clip.py index 9dad78eb163b..4f7177b8f0bb 100644 --- a/examples/tensorflow/contrastive-image-text/run_clip.py +++ b/examples/tensorflow/contrastive-image-text/run_clip.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version( "datasets>=1.8.0", "To fix: pip install -r examples/tensorflow/contrastive-image-text/requirements.txt" diff --git a/examples/tensorflow/image-classification/run_image_classification.py b/examples/tensorflow/image-classification/run_image_classification.py index b115906064f4..7e68a02986ef 100644 --- a/examples/tensorflow/image-classification/run_image_classification.py +++ b/examples/tensorflow/image-classification/run_image_classification.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.24.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/image-classification/requirements.txt") diff --git a/examples/tensorflow/multiple-choice/run_swag.py b/examples/tensorflow/multiple-choice/run_swag.py index 27b3a5518bb0..d4b5955b408c 100644 --- a/examples/tensorflow/multiple-choice/run_swag.py +++ b/examples/tensorflow/multiple-choice/run_swag.py @@ -50,7 +50,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = logging.getLogger(__name__) diff --git a/examples/tensorflow/question-answering/run_qa.py b/examples/tensorflow/question-answering/run_qa.py index dd39a6cd6a41..bcc7b24f2334 100755 --- a/examples/tensorflow/question-answering/run_qa.py +++ b/examples/tensorflow/question-answering/run_qa.py @@ -48,7 +48,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") logger = logging.getLogger(__name__) diff --git a/examples/tensorflow/summarization/run_summarization.py b/examples/tensorflow/summarization/run_summarization.py index 3295d77853ea..e2b19746d807 100644 --- a/examples/tensorflow/summarization/run_summarization.py +++ b/examples/tensorflow/summarization/run_summarization.py @@ -53,7 +53,7 @@ # region Checking dependencies # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/summarization/requirements.txt") diff --git a/examples/tensorflow/text-classification/run_glue.py b/examples/tensorflow/text-classification/run_glue.py index 5208400c9e3b..09936a319000 100644 --- a/examples/tensorflow/text-classification/run_glue.py +++ b/examples/tensorflow/text-classification/run_glue.py @@ -47,7 +47,7 @@ # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") task_to_keys = { "cola": ("sentence", None), diff --git a/examples/tensorflow/translation/run_translation.py b/examples/tensorflow/translation/run_translation.py index 4622601f8b11..43f13bd0a561 100644 --- a/examples/tensorflow/translation/run_translation.py +++ b/examples/tensorflow/translation/run_translation.py @@ -56,7 +56,7 @@ # region Dependencies and constants # Will error if the minimal version of Transformers is not installed. Remove at your own risks. -check_min_version("4.27.0.dev0") +check_min_version("4.28.0.dev0") require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/summarization/requirements.txt") diff --git a/setup.py b/setup.py index 1be88908c937..943bb196b5d6 100644 --- a/setup.py +++ b/setup.py @@ -418,7 +418,7 @@ def run(self): setup( name="transformers", - version="4.27.0.dev0", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots) + version="4.28.0.dev0", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots) author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)", author_email="transformers@huggingface.co", description="State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow", diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 45fe5447b31b..f0add2263abc 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -18,7 +18,7 @@ # to defer the actual importing for when the objects are requested. This way `import transformers` provides the names # in the namespace without actually importing anything (and especially none of the backends). -__version__ = "4.27.0.dev0" +__version__ = "4.28.0.dev0" from typing import TYPE_CHECKING From 741ec6b934eb18f41d4f694548e86e05591ab767 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:30:32 -0400 Subject: [PATCH 100/277] Load optimizer state on CPU to avoid CUDA OOM (#22159) --- src/transformers/trainer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 2afa1d679b0f..efd8d0fc3a4e 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -2416,7 +2416,6 @@ def _load_optimizer_and_scheduler(self, checkpoint): self.optimizer.load_state_dict(optimizer_state) self.lr_scheduler.load_state_dict(lr_scheduler_state) else: - map_location = "cpu" if is_sagemaker_mp_enabled() else self.args.device if is_sagemaker_mp_enabled(): if os.path.isfile(os.path.join(checkpoint, "user_content.pt")): # Optimizer checkpoint was saved with smp >= 1.10 @@ -2436,7 +2435,7 @@ def opt_load_hook(mod, opt): self.model_wrapped.register_post_step_hook(opt_load_hook) else: self.optimizer.load_state_dict( - torch.load(os.path.join(checkpoint, OPTIMIZER_NAME), map_location=map_location) + torch.load(os.path.join(checkpoint, OPTIMIZER_NAME), map_location="cpu") ) with warnings.catch_warnings(record=True) as caught_warnings: self.lr_scheduler.load_state_dict(torch.load(os.path.join(checkpoint, SCHEDULER_NAME))) From 350740e10414ead2ebb2f647bcb862db6bf84dfa Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:30:43 -0400 Subject: [PATCH 101/277] Run all tests by default (#22162) --- .circleci/create_circleci_config.py | 9 ++++++++- src/transformers/testing_utils.py | 15 ++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 8fa5e3e70d7b..d67968d5afec 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -24,7 +24,14 @@ import yaml -COMMON_ENV_VARIABLES = {"OMP_NUM_THREADS": 1, "TRANSFORMERS_IS_CI": True, "PYTEST_TIMEOUT": 120, "RUN_PIPELINE_TESTS": False} +COMMON_ENV_VARIABLES = { + "OMP_NUM_THREADS": 1, + "TRANSFORMERS_IS_CI": True, + "PYTEST_TIMEOUT": 120, + "RUN_PIPELINE_TESTS": False, + "RUN_PT_TF_CROSS_TESTS": False, + "RUN_PT_FLAX_CROSS_TESTS": False, +} COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile", "s": None} DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.7.12"}] diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index db744e2fb3a0..3fd334d71cb1 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -139,11 +139,10 @@ def parse_int_from_env(key, default=None): _run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) -_run_pt_tf_cross_tests = parse_flag_from_env("RUN_PT_TF_CROSS_TESTS", default=False) -_run_pt_flax_cross_tests = parse_flag_from_env("RUN_PT_FLAX_CROSS_TESTS", default=False) +_run_pt_tf_cross_tests = parse_flag_from_env("RUN_PT_TF_CROSS_TESTS", default=True) +_run_pt_flax_cross_tests = parse_flag_from_env("RUN_PT_FLAX_CROSS_TESTS", default=True) _run_custom_tokenizers = parse_flag_from_env("RUN_CUSTOM_TOKENIZERS", default=False) _run_staging = parse_flag_from_env("HUGGINGFACE_CO_STAGING", default=False) -_run_git_lfs_tests = parse_flag_from_env("RUN_GIT_LFS_TESTS", default=False) _tf_gpu_memory_limit = parse_int_from_env("TF_GPU_MEMORY_LIMIT", default=None) _run_pipeline_tests = parse_flag_from_env("RUN_PIPELINE_TESTS", default=True) @@ -257,16 +256,6 @@ def require_bs4(test_case): return unittest.skipUnless(is_bs4_available(), "test requires BeautifulSoup4")(test_case) -def require_git_lfs(test_case): - """ - Decorator marking a test that requires git-lfs. - - git-lfs requires additional dependencies, and tests are skipped by default. Set the RUN_GIT_LFS_TESTS environment - variable to a truthy value to run them. - """ - return unittest.skipUnless(_run_git_lfs_tests, "test of git lfs workflow")(test_case) - - def require_accelerate(test_case): """ Decorator marking a test that requires accelerate. These tests are skipped when accelerate isn't installed. From 81b794fe266e976e6d1dd04391f27db9a05858b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=AE=E8=BA=81=E7=9A=84=E5=B0=8F=E8=9E=83=E8=9F=B9?= <876464857@qq.com> Date: Thu, 16 Mar 2023 00:27:19 +0800 Subject: [PATCH 102/277] Fix: unfinished_sequences with correct device (#22184) Fix: unfinished_sequences with correct device The original code was causing errors when running torch.jit.trace due to the tensor options being incorrect. I fixed this by using torch.ones to create a tensor with the correct device and dtype. This should resolve the issue with running torch.jit.trace. --- src/transformers/generation/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index ad18644ee5cb..b0b4e577d3c3 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -1805,7 +1805,7 @@ def contrastive_search( ) # keep track of which sequences are already finished - unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1) + unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device) this_peer_finished = False # used by synced_gpus only batch_size = input_ids.shape[0] @@ -2180,7 +2180,7 @@ def greedy_search( ) # keep track of which sequences are already finished - unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1) + unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device) this_peer_finished = False # used by synced_gpus only while True: @@ -2446,7 +2446,7 @@ def sample( ) # keep track of which sequences are already finished - unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1) + unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device) this_peer_finished = False # used by synced_gpus only # auto-regressive generation From 3704f822ce1b1c1939b28f991136c476c159b6df Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Wed, 15 Mar 2023 17:37:23 +0000 Subject: [PATCH 103/277] Revert 22152 MaskedImageCompletionOutput changes (#22187) Revert changes --- src/transformers/modeling_outputs.py | 28 --------------------- src/transformers/models/vit/modeling_vit.py | 15 ++++------- tests/models/vit/test_modeling_vit.py | 4 +-- 3 files changed, 7 insertions(+), 40 deletions(-) diff --git a/src/transformers/modeling_outputs.py b/src/transformers/modeling_outputs.py index a4d465b1d48f..4f7540d0ff9e 100755 --- a/src/transformers/modeling_outputs.py +++ b/src/transformers/modeling_outputs.py @@ -1281,34 +1281,6 @@ class ImageSuperResolutionOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -@dataclass -class MaskedImageCompletionOutput(ModelOutput): - """ - Base class for outputs of masked image completion / in-painting models. - - Args: - loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): - Reconstruction loss. - reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): - Reconstructed / completed images. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + - one for the output of each stage) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states - (also called feature maps) of the model at the output of each stage. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - """ - - loss: Optional[torch.FloatTensor] = None - reconstruction: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - attentions: Optional[Tuple[torch.FloatTensor]] = None - - @dataclass class Wav2Vec2BaseModelOutput(ModelOutput): """ diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index 154afdb211f3..449eda3ee821 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -25,12 +25,7 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import ( - BaseModelOutput, - BaseModelOutputWithPooling, - ImageClassifierOutput, - MaskedImageCompletionOutput, -) +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput, MaskedLMOutput from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( @@ -648,7 +643,7 @@ def __init__(self, config: ViTConfig) -> None: self.post_init() @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedImageCompletionOutput, config_class=_CONFIG_FOR_DOC) + @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -658,7 +653,7 @@ def forward( output_hidden_states: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[tuple, MaskedImageCompletionOutput]: + ) -> Union[tuple, MaskedLMOutput]: r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). @@ -728,9 +723,9 @@ def forward( output = (reconstructed_pixel_values,) + outputs[1:] return ((masked_im_loss,) + output) if masked_im_loss is not None else output - return MaskedImageCompletionOutput( + return MaskedLMOutput( loss=masked_im_loss, - reconstruction=reconstructed_pixel_values, + logits=reconstructed_pixel_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) diff --git a/tests/models/vit/test_modeling_vit.py b/tests/models/vit/test_modeling_vit.py index d7ae3c162f59..be509d460e63 100644 --- a/tests/models/vit/test_modeling_vit.py +++ b/tests/models/vit/test_modeling_vit.py @@ -134,7 +134,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label model.eval() result = model(pixel_values) self.parent.assertEqual( - result.reconstruction.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) + result.logits.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) ) # test greyscale images @@ -145,7 +145,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size]) result = model(pixel_values) - self.parent.assertEqual(result.reconstruction.shape, (self.batch_size, 1, self.image_size, self.image_size)) + self.parent.assertEqual(result.logits.shape, (self.batch_size, 1, self.image_size, self.image_size)) def create_and_check_for_image_classification(self, config, pixel_values, labels): config.num_labels = self.type_sequence_label_size From f4c75517b46a3073f3ecee999e61b8abed280001 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 15 Mar 2023 14:13:38 -0400 Subject: [PATCH 104/277] Regression pipeline device (#22190) * Fix regression in pipeline when device=-1 is passed * Add regression test --- src/transformers/pipelines/base.py | 4 ++-- tests/pipelines/test_pipelines_common.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py index 3dca2d33d157..cdb597ef9661 100644 --- a/src/transformers/pipelines/base.py +++ b/src/transformers/pipelines/base.py @@ -769,8 +769,8 @@ def __init__( self.modelcard = modelcard self.framework = framework - if self.framework == "pt" and device is not None: - self.model = self.model.to(device=device) + if self.framework == "pt" and device is not None and not (isinstance(device, int) and device < 0): + self.model.to(device) if device is None: # `accelerate` device map diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index f43f439ac279..1a3ddace2871 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -484,6 +484,14 @@ def add(number, extra=0): outputs = list(dataset) self.assertEqual(outputs, [[{"id": 2}, {"id": 3}, {"id": 4}, {"id": 5}]]) + def test_pipeline_negative_device(self): + # To avoid regressing, pipeline used to accept device=-1 + classifier = pipeline("text-generation", "hf-internal-testing/tiny-random-bert", device=-1) + + expected_output = [{"generated_text": ANY(str)}] + actual_output = classifier("Test input.") + self.assertEqual(expected_output, actual_output) + @slow @require_torch def test_load_default_pipelines_pt(self): From 12225357fcf170e57b5bc6bcd1c2aef526be2fa7 Mon Sep 17 00:00:00 2001 From: Anahita Bhiwandiwalla Date: Wed, 15 Mar 2023 12:54:38 -0700 Subject: [PATCH 105/277] Update BridgeTowerForContrastiveLearning (#22145) * Use return_loss for BridgeTowerForContrastiveLearning, add example * fix tests * Update example in BridgeTowerForContrastiveLearning * Update test_modeling_bridgetower.py * update model output format * minor update * Update src/transformers/models/bridgetower/modeling_bridgetower.py * make style --------- Co-authored-by: Tiep Le <97980157+tileintel@users.noreply.github.com> Co-authored-by: Tiep Le Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Co-authored-by: ydshieh --- .../bridgetower/modeling_bridgetower.py | 58 ++++++++++++------- .../bridgetower/test_modeling_bridgetower.py | 16 ++--- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/transformers/models/bridgetower/modeling_bridgetower.py b/src/transformers/models/bridgetower/modeling_bridgetower.py index f405407d7d9b..209ff9703ff2 100644 --- a/src/transformers/models/bridgetower/modeling_bridgetower.py +++ b/src/transformers/models/bridgetower/modeling_bridgetower.py @@ -166,6 +166,8 @@ class BridgeTowerContrastiveOutput(ModelOutput): Output type of ['BridgeTowerForContrastiveLearning'] Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`: + Image-text contrastive loss. logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). text_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`): @@ -174,24 +176,22 @@ class BridgeTowerContrastiveOutput(ModelOutput): The image embeddings obtained by applying the projection layer to the pooler_output. cross_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`): The text-image cross-modal embeddings obtained by applying the projection layer to the pooler_output. - loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): - Image-text contrastive loss. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. """ + loss: Optional[torch.FloatTensor] = None logits: torch.FloatTensor = None text_embeds: Optional[Tuple[torch.FloatTensor]] = None image_embeds: Optional[Tuple[torch.FloatTensor]] = None cross_embeds: Optional[Tuple[torch.FloatTensor]] = None - loss: Optional[torch.FloatTensor] = None - attentions: Optional[Tuple[torch.FloatTensor]] = None hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None class BridgeTowerResidualAttention(nn.Module): @@ -1789,12 +1789,11 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = True, return_dict: Optional[bool] = None, - labels: Optional[torch.LongTensor] = None, + return_loss: Optional[bool] = None, ) -> Union[BridgeTowerContrastiveOutput, Tuple[torch.FloatTensor]]: r""" - labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*): - Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match. - The pairs with 0 will be skipped for calculation. + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. Returns: Examples: @@ -1803,14 +1802,29 @@ def forward( >>> from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning >>> import requests >>> from PIL import Image + >>> import torch - >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" - >>> image = Image.open(requests.get(url, stream=True).raw) - >>> texts = "An image of two cats chilling on a couch" + >>> image_urls = [ + ... "https://farm4.staticflickr.com/3395/3428278415_81c3e27f15_z.jpg", + ... "http://images.cocodataset.org/val2017/000000039769.jpg", + ... ] + >>> texts = ["two dogs in a car", "two cats sleeping on a couch"] + >>> images = [Image.open(requests.get(url, stream=True).raw) for url in image_urls] >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") >>> model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") - >>> outputs = model(**inputs, output_hidden_states=True) + + >>> inputs = processor(images, texts, padding=True, return_tensors="pt") + >>> loss = model(**inputs, return_loss=True).loss + + >>> inputs = processor(images, texts[::-1], padding=True, return_tensors="pt") + >>> loss_swapped = model(**inputs, return_loss=True).loss + + >>> print("Loss", round(loss.item(), 4)) + Loss 0.0019 + + >>> print("Loss with swapped images", round(loss_swapped.item(), 4)) + Loss with swapped images 2.126 ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1857,23 +1871,23 @@ def forward( itc_loss = None - if labels is not None: - labels = torch.arange(len(labels), device=logits.device) + if return_loss: + labels = torch.arange(len(logits), device=logits.device) text_to_image_loss = nn.functional.cross_entropy(logits_text_to_image, labels) text_to_cross_loss = nn.functional.cross_entropy(logits_text_to_cross, labels) image_to_cross_loss = nn.functional.cross_entropy(logits_image_to_cross, labels) itc_loss = (text_to_image_loss + text_to_cross_loss + image_to_cross_loss) / 3.0 if not return_dict: - output = tuple(logits) + output = (logits, text_embeds, image_embeds, cross_embeds) + outputs[3:] return ((itc_loss,) + output) if itc_loss is not None else output return BridgeTowerContrastiveOutput( - attentions=outputs.attentions, - hidden_states=outputs.hidden_states, + loss=itc_loss, + logits=logits, text_embeds=text_embeds, image_embeds=image_embeds, cross_embeds=cross_embeds, - logits=logits, - loss=itc_loss, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, ) diff --git a/tests/models/bridgetower/test_modeling_bridgetower.py b/tests/models/bridgetower/test_modeling_bridgetower.py index 20396c8bf7bf..078d66b51645 100644 --- a/tests/models/bridgetower/test_modeling_bridgetower.py +++ b/tests/models/bridgetower/test_modeling_bridgetower.py @@ -94,7 +94,7 @@ def __init__( self.num_hidden_layers = num_hidden_layers self.tie_word_embeddings = tie_word_embeddings self.init_layernorm_from_vision_encoder = init_layernorm_from_vision_encoder - self.vocab_size = 50265 + self.vocab_size = 99 self.num_channels = 3 self.seq_length = 4 self.num_image_features = 325 @@ -188,7 +188,7 @@ def create_and_check_for_masked_language_modeling( result = model(input_ids, attention_mask=attention_mask, pixel_values=pixel_values, pixel_mask=pixel_mask) result = model(input_ids, attention_mask=attention_mask, pixel_values=pixel_values) - self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, 50265)) def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() @@ -231,7 +231,7 @@ def extract_output(self, outputs, model_class): def setUp(self): self.model_tester = BridgeTowerModelTester(self) - self.config_tester = ConfigTester(self, config_class=BridgeTowerConfig, hidden_size=37, vocab_size=50265) + self.config_tester = ConfigTester(self, config_class=BridgeTowerConfig, hidden_size=37, vocab_size=99) def test_config(self): self.config_tester.run_common_tests() @@ -486,9 +486,9 @@ def test_constrastive_learning(self): processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc") image = prepare_img() text = "a bunch of cats laying on a tower." - inputs = processor(image, text, return_tensors="pt").to(torch_device) + inputs = processor(image, text, padding=True, return_tensors="pt").to(torch_device) with torch.no_grad(): - outputs = model(**inputs, output_hidden_states=True) + outputs = model(**inputs, output_hidden_states=True, return_loss=True) # verify the logits expected_shape = torch.Size([1, 3, 512]) @@ -507,14 +507,16 @@ class BridgeTowerModelTrainingTest(unittest.TestCase): def setUp(self): self.model_tester = BridgeTowerModelTester(self) - self.config_tester = ConfigTester(self, config_class=BridgeTowerConfig, hidden_size=37, vocab_size=50265) + self.config_tester = ConfigTester(self, config_class=BridgeTowerConfig, hidden_size=37, vocab_size=99) def _prepare_inputs_for_training(self, model_class): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() if model_class == BridgeTowerForMaskedLM: inputs_dict["labels"] = inputs_dict["input_ids"] - elif model_class == BridgeTowerForImageAndTextRetrieval or model_class == BridgeTowerForContrastiveLearning: + elif model_class == BridgeTowerForImageAndTextRetrieval: inputs_dict["labels"] = ids_tensor([1], 2) + elif model_class == BridgeTowerForContrastiveLearning: + inputs_dict["return_loss"] = True return config, inputs_dict def _get_non_used_layer_names(self, model_class): From 860a3cd0fc2b78b1baf5581fed0f3be749e8e170 Mon Sep 17 00:00:00 2001 From: Prathik Rao Date: Wed, 15 Mar 2023 13:11:15 -0700 Subject: [PATCH 106/277] t5 remove data dependency (#22097) * t5 remove data dependency * make style * make fix-copies --------- Co-authored-by: Prathik Rao --- src/transformers/models/mt5/modeling_mt5.py | 24 +++++++++++++++------ src/transformers/models/t5/modeling_t5.py | 24 +++++++++++++++------ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/transformers/models/mt5/modeling_mt5.py b/src/transformers/models/mt5/modeling_mt5.py index e2235fc69055..43beb69f10f2 100644 --- a/src/transformers/models/mt5/modeling_mt5.py +++ b/src/transformers/models/mt5/modeling_mt5.py @@ -566,8 +566,12 @@ def forward( attention_outputs = self_attention_outputs[2:] # Keep self-attention outputs and relative position weights # clamp inf values to enable fp16 training - if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): - clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) do_cross_attention = self.is_decoder and encoder_hidden_states is not None @@ -593,8 +597,12 @@ def forward( hidden_states = cross_attention_outputs[0] # clamp inf values to enable fp16 training - if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): - clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) # Combine self attn and cross attn key value states @@ -608,8 +616,12 @@ def forward( hidden_states = self.layer[-1](hidden_states) # clamp inf values to enable fp16 training - if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): - clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) outputs = (hidden_states,) diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py index 19cb83dac352..c056ef73cc1f 100644 --- a/src/transformers/models/t5/modeling_t5.py +++ b/src/transformers/models/t5/modeling_t5.py @@ -703,8 +703,12 @@ def forward( attention_outputs = self_attention_outputs[2:] # Keep self-attention outputs and relative position weights # clamp inf values to enable fp16 training - if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): - clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) do_cross_attention = self.is_decoder and encoder_hidden_states is not None @@ -730,8 +734,12 @@ def forward( hidden_states = cross_attention_outputs[0] # clamp inf values to enable fp16 training - if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): - clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) # Combine self attn and cross attn key value states @@ -745,8 +753,12 @@ def forward( hidden_states = self.layer[-1](hidden_states) # clamp inf values to enable fp16 training - if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): - clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) outputs = (hidden_states,) From cd0d499ab358223ae6b4659ded75eabbdc442638 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 16 Mar 2023 05:52:40 +0100 Subject: [PATCH 107/277] Fix DeepSpeed CI (#22194) * Deal with torch-tensorrt --------- Co-authored-by: ydshieh --- docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile index 1c20ee261be3..2fa1317f8275 100644 --- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile +++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile @@ -24,9 +24,8 @@ RUN python3 -m pip install --no-cache-dir -U torch torchvision torchaudio --inde RUN python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing] -# This will uninstall torch 2.0.0 -# TODO: uncomment the following line once `torch-tensorrt` is ready for `torch 2.0.0` -# RUN python3 -m pip install torch-tensorrt==1.3.0 --find-links https://github.com/pytorch/TensorRT/releases/expanded_assets/v1.3.0 +# Uninstall `torch-tensorrt` shipped with the base image +RUN python3 -m pip uninstall -y torch-tensorrt # recompile apex RUN python3 -m pip uninstall -y apex From 6ac5880ae7695c394fdefafd50cee766a089477a Mon Sep 17 00:00:00 2001 From: Alara Dirik <8944735+alaradirik@users.noreply.github.com> Date: Thu, 16 Mar 2023 13:41:48 +0300 Subject: [PATCH 108/277] Fix typo in Align docs (#22199) Fix align docs typo --- docs/source/en/model_doc/align.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/model_doc/align.mdx b/docs/source/en/model_doc/align.mdx index 043de683c17a..159c3b2a0d05 100644 --- a/docs/source/en/model_doc/align.mdx +++ b/docs/source/en/model_doc/align.mdx @@ -59,7 +59,7 @@ The original code is not released, this implementation is based on the Kakao Bra A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with ALIGN. -- A blog post on [ALIGN and the COYO-700M dataset](https://huggingface.co/blog). +- A blog post on [ALIGN and the COYO-700M dataset](https://huggingface.co/blog/vit-align). - A zero-shot image classification [demo](https://huggingface.co/spaces/adirik/ALIGN-zero-shot-image-classification). - [Model card](https://huggingface.co/kakaobrain/align-base) of `kakaobrain/align-base` model. From 3aa4b1dc062f46f58948815328191fd3b711ea1d Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 16 Mar 2023 12:48:52 +0100 Subject: [PATCH 109/277] Update expected values in `MgpstrModelIntegrationTest` (#22195) Update values Co-authored-by: ydshieh --- tests/models/mgp_str/test_modeling_mgp_str.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/mgp_str/test_modeling_mgp_str.py b/tests/models/mgp_str/test_modeling_mgp_str.py index ab5b6422b627..990f78587da8 100644 --- a/tests/models/mgp_str/test_modeling_mgp_str.py +++ b/tests/models/mgp_str/test_modeling_mgp_str.py @@ -262,7 +262,7 @@ def test_inference(self): self.assertEqual(out_strs["generated_text"][0], expected_text) expected_slice = torch.tensor( - [[[-39.7358, -44.8562, -36.6253], [-62.3605, -64.5908, -59.0069], [-74.6127, -68.9724, -71.7150]]], + [[[-39.5397, -44.4024, -36.1844], [-61.4709, -63.8639, -58.3454], [-74.0225, -68.5494, -71.2164]]], device=torch_device, ) From bd9f26a8fc4fa4b3826cd68dab06e118c8fbfe39 Mon Sep 17 00:00:00 2001 From: Baelish03 <97971495+Baelish03@users.noreply.github.com> Date: Thu, 16 Mar 2023 13:00:07 +0100 Subject: [PATCH 110/277] Italian Translation of migration.mdx (#22183) * Tranlstion Italian: migration * Update migration.mdx minor fixes * Update _toctree.yml * Delete migration.mdx * Add italian translation of migration.mdx * Update of migration.mdx translation and toctree --- docs/source/it/_toctree.yml | 116 ++++++------- docs/source/it/migration.mdx | 316 +++++++++++++++++++++++++++++++++++ 2 files changed, 375 insertions(+), 57 deletions(-) create mode 100644 docs/source/it/migration.mdx diff --git a/docs/source/it/_toctree.yml b/docs/source/it/_toctree.yml index c0f4fd082491..abb252b321b7 100644 --- a/docs/source/it/_toctree.yml +++ b/docs/source/it/_toctree.yml @@ -1,57 +1,59 @@ -- sections: - - local: index - title: 🤗 Transformers - - local: quicktour - title: Tour rapido - - local: installation - title: Installazione - title: Iniziare -- sections: - - local: pipeline_tutorial - title: Pipeline per l'inferenza - - local: autoclass_tutorial - title: Carica istanze pre-allenate con AutoClass - - local: preprocessing - title: Preprocess - - local: training - title: Fine-tuning di un modello pre-addestrato - - local: accelerate - title: Allenamento distribuito con 🤗 Accelerate - - local: model_sharing - title: Condividere un modello - title: Esercitazione -- sections: - - local: create_a_model - title: Crea un'architettura personalizzata - - local: custom_models - title: Condividere modelli personalizzati - - local: run_scripts - title: Addestramento con script - - local: multilingual - title: Modelli multilingua per l'inferenza - - local: converting_tensorflow_models - title: Convertire modelli tensorflow - - local: serialization - title: Esporta modelli Transformers - - local: perf_train_cpu - title: Addestramento efficiente su CPU - - local: perf_train_cpu_many - title: Addestramento efficiente su multiple CPU - - local: big_models - title: Istanziare un big model - - local: debugging - title: Debugging - title: Guide pratiche -- sections: - - local: add_new_pipeline - title: Come aggiungere una pipeline a 🤗 Transformers? - - local: add_new_model - title: Come aggiungere un modello a 🤗 Transformers? - - local: perf_hardware - title: Hardware ottimizzato per l'addestramento - - local: community - title: Risorse della comunità - - local: pr_checks - title: Controlli su una Pull Request - title: Guide How-to - +- sections: + - local: index + title: 🤗 Transformers + - local: quicktour + title: Tour rapido + - local: installation + title: Installazione + title: Iniziare +- sections: + - local: pipeline_tutorial + title: Pipeline per l'inferenza + - local: autoclass_tutorial + title: Carica istanze pre-allenate con AutoClass + - local: preprocessing + title: Preprocess + - local: training + title: Fine-tuning di un modello pre-addestrato + - local: accelerate + title: Allenamento distribuito con 🤗 Accelerate + - local: model_sharing + title: Condividere un modello + title: Esercitazione +- sections: + - local: create_a_model + title: Crea un'architettura personalizzata + - local: custom_models + title: Condividere modelli personalizzati + - local: run_scripts + title: Addestramento con script + - local: multilingual + title: Modelli multilingua per l'inferenza + - local: converting_tensorflow_models + title: Convertire modelli tensorflow + - local: serialization + title: Esporta modelli Transformers + - local: perf_train_cpu + title: Addestramento efficiente su CPU + - local: perf_train_cpu_many + title: Addestramento efficiente su multiple CPU + - local: big_models + title: Istanziare un big model + - local: migration + title: Passaggio da pacchetti precedenti + - local: debugging + title: Debugging + title: Guide pratiche +- sections: + - local: add_new_pipeline + title: Come aggiungere una pipeline a 🤗 Transformers? + - local: add_new_model + title: Come aggiungere un modello a 🤗 Transformers? + - local: perf_hardware + title: Hardware ottimizzato per l'addestramento + - local: community + title: Risorse della comunità + - local: pr_checks + title: Controlli su una Pull Request + title: Guide How-to + diff --git a/docs/source/it/migration.mdx b/docs/source/it/migration.mdx new file mode 100644 index 000000000000..92622787c6e9 --- /dev/null +++ b/docs/source/it/migration.mdx @@ -0,0 +1,316 @@ + + +# Migrazione da pacchetti precedenti + +## Migrazione da transformers `v3.x` a `v4.x` + +Un paio di modifiche sono state introdotte nel passaggio dalla versione 3 alla versione 4. Di seguito è riportato un riepilogo delle +modifiche previste: + +#### 1. AutoTokenizer e pipeline ora utilizzano tokenizer veloci (rust) per impostazione predefinita. + +I tokenizer python e rust hanno all'incirca le stesse API, ma i tokenizer rust hanno un set di funzionalità più completo. + +Ciò introduce due modifiche sostanziali: +- La gestione dei token in overflow tra i tokenizer Python e Rust è diversa. +- I tokenizers di rust non accettano numeri interi nei metodi di codifica. + +##### Come ottenere lo stesso comportamento di v3.x in v4.x + +- Le pipeline ora contengono funzionalità aggiuntive pronte all'uso. Vedi la [pipeline di classificazione dei token con il flag `grouped_entities`](main_classes/pipelines#transformers.TokenClassificationPipeline). +- Gli auto-tokenizer ora restituiscono tokenizer rust. Per ottenere invece i tokenizer python, l'utente deve usare il flag `use_fast` impostandolo `False`: + +Nella versione `v3.x`: +```py +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") +``` +per ottenere lo stesso nella versione `v4.x`: +```py +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("bert-base-cased", use_fast=False) +``` + +#### 2. SentencePiece è stato rimosso dalle dipendenze richieste + +Il requisito sulla dipendenza SentencePiece è stato rimosso da `setup.py`. È stato fatto per avere un canale su anaconda cloud senza basarsi su `conda-forge`. Ciò significa che i tokenizer che dipendono dalla libreria SentencePiece non saranno disponibili con un'installazione standard di `transformers`. + +Ciò include le versioni **lente** di: +- `XLNetTokenizer` +- `AlbertTokenizer` +- `CamembertTokenizer` +- `MBartTokenizer` +- `PegasusTokenizer` +- `T5Tokenizer` +- `ReformerTokenizer` +- `XLMRobertaTokenizer` + +##### Come ottenere lo stesso comportamento della v3.x nella v4.x + +Per ottenere lo stesso comportamento della versione `v3.x`, devi installare anche `sentencepiece`: + +Nella versione `v3.x`: +```bash +pip install transformers +``` +per ottenere lo stesso nella versione `v4.x`: +```bash +pip install transformers[sentencepiece] +``` +o +```bash +pip install transformers stentencepiece +``` +#### 3. L'architettura delle repo è stato aggiornata in modo che ogni modello abbia la propria cartella + +Con l’aggiunta di nuovi modelli, il numero di file nella cartella `src/transformers` continua a crescere e diventa più difficile navigare e capire. Abbiamo fatto la scelta di inserire ogni modello e i file che lo accompagnano nelle proprie sottocartelle. + +Si tratta di una modifica sostanziale in quanto l'importazione di layer intermedi utilizzando direttamente il modulo di un modello deve essere eseguita tramite un percorso diverso. + +##### Come ottenere lo stesso comportamento della v3.x nella v4.x + +Per ottenere lo stesso comportamento della versione `v3.x`, devi aggiornare il percorso utilizzato per accedere ai layer. + +Nella versione `v3.x`: +```bash +from transformers.modeling_bert import BertLayer +``` +per ottenere lo stesso nella versione `v4.x`: +```bash +from transformers.models.bert.modeling_bert import BertLayer +``` + +#### 4. Impostare l'argomento `return_dict` su `True` per impostazione predefinita + +L'[argomento `return_dict`](main_classes/output) abilita la restituzione di oggetti python dict-like contenenti gli output del modello, invece delle tuple standard. Questo oggetto è self-documented poiché le chiavi possono essere utilizzate per recuperare valori, comportandosi anche come una tupla e gli utenti possono recuperare oggetti per indexing o slicing. + +Questa è una modifica sostanziale poiché la tupla non può essere decompressa: `value0, value1 = outputs` non funzionerà. + +##### Come ottenere lo stesso comportamento della v3.x nella v4.x + +Per ottenere lo stesso comportamento della versione `v3.x`, specifica l'argomento `return_dict` come `False`, sia nella configurazione del modello che nel passaggio successivo. + +Nella versione `v3.x`: +```bash +model = BertModel.from_pretrained("bert-base-cased") +outputs = model(**inputs) +``` +per ottenere lo stesso nella versione `v4.x`: +```bash +model = BertModel.from_pretrained("bert-base-cased") +outputs = model(**inputs, return_dict=False) +``` +o +```bash +model = BertModel.from_pretrained("bert-base-cased", return_dict=False) +outputs = model(**inputs) +``` + +#### 5. Rimozione di alcuni attributi deprecati + +Gli attributi sono stati rimossi se deprecati da almeno un mese. L'elenco completo degli attributi obsoleti è disponibile in [#8604](https://github.com/huggingface/transformers/pull/8604). + +Ecco un elenco di questi attributi/metodi/argomenti e quali dovrebbero essere le loro sostituzioni: + +In diversi modelli, le etichette diventano coerenti con gli altri modelli: +- `masked_lm_labels` diventa `labels` in `AlbertForMaskedLM` e `AlbertForPreTraining`. +- `masked_lm_labels` diventa `labels` in `BertForMaskedLM` e `BertForPreTraining`. +- `masked_lm_labels` diventa `labels` in `DistilBertForMaskedLM`. +- `masked_lm_labels` diventa `labels` in `ElectraForMaskedLM`. +- `masked_lm_labels` diventa `labels` in `LongformerForMaskedLM`. +- `masked_lm_labels` diventa `labels` in `MobileBertForMaskedLM`. +- `masked_lm_labels` diventa `labels` in `RobertaForMaskedLM`. +- `lm_labels` diventa `labels` in `BartForConditionalGeneration`. +- `lm_labels` diventa `labels` in `GPT2DoubleHeadsModel`. +- `lm_labels` diventa `labels` in `OpenAIGPTDoubleHeadsModel`. +- `lm_labels` diventa `labels` in `T5ForConditionalGeneration`. + +In diversi modelli, il meccanismo di memorizzazione nella cache diventa coerente con gli altri: +- `decoder_cached_states` diventa `past_key_values` in tutti i modelli BART-like, FSMT e T5. +- `decoder_past_key_values` diventa `past_key_values` in tutti i modelli BART-like, FSMT e T5. +- `past` diventa `past_key_values` in tutti i modelli CTRL. +- `past` diventa `past_key_values` in tutti i modelli GPT-2. + +Per quanto riguarda le classi tokenizer: +- L'attributo tokenizer `max_len` diventa `model_max_length`. +- L'attributo tokenizer `return_lengths` diventa `return_length`. +- L'argomento di codifica del tokenizer `is_pretokenized` diventa `is_split_into_words`. + +Per quanto riguarda la classe `Trainer`: +- L'argomento `tb_writer` di `Trainer` è stato rimosso in favore della funzione richiamabile `TensorBoardCallback(tb_writer=...)`. +- L'argomento `prediction_loss_only` di `Trainer` è stato rimosso in favore dell'argomento di classe `args.prediction_loss_only`. +- L'attributo `data_collator` di `Trainer` sarà richiamabile. +- Il metodo `_log` di `Trainer` è deprecato a favore di `log`. +- Il metodo `_training_step` di `Trainer` è deprecato a favore di `training_step`. +- Il metodo `_prediction_loop` di `Trainer` è deprecato a favore di `prediction_loop`. +- Il metodo `is_local_master` di `Trainer` è deprecato a favore di `is_local_process_zero`. +- Il metodo `is_world_master` di `Trainer` è deprecato a favore di `is_world_process_zero`. + +Per quanto riguarda la classe `TFTrainer`: +- L'argomento `prediction_loss_only` di `TFTrainer` è stato rimosso a favore dell'argomento di classe `args.prediction_loss_only`. +- Il metodo `_log` di `Trainer` è deprecato a favore di `log`. +- Il metodo `_prediction_loop` di `TFTrainer` è deprecato a favore di `prediction_loop`. +- Il metodo `_setup_wandb` di `TFTrainer` è deprecato a favore di `setup_wandb`. +- Il metodo `_run_model` di `TFTrainer` è deprecato a favore di `run_model`. + +Per quanto riguarda la classe `TrainingArguments`: +- L'argomento `evaluate_during_training` di `TrainingArguments` è deprecato a favore di `evaluation_strategy`. + +Per quanto riguarda il modello Transfo-XL: +- L'attributo di configurazione `tie_weight` di Transfo-XL diventa `tie_words_embeddings`. +- Il metodo di modellazione `reset_length` di Transfo-XL diventa `reset_memory_length`. + +Per quanto riguarda le pipeline: +- L'argomento `topk` di `FillMaskPipeline` diventa `top_k`. + + + +## Passaggio da pytorch-transformers a 🤗 Transformers + +Ecco un breve riepilogo di ciò a cui prestare attenzione durante il passaggio da `pytorch-transformers` a 🤗 Transformers. + +### L’ordine posizionale di alcune parole chiave di input dei modelli (`attention_mask`, `token_type_ids`...) è cambiato + +Per usare Torchscript (vedi #1010, #1204 e #1195) l'ordine specifico delle **parole chiave di input** di alcuni modelli (`attention_mask`, `token_type_ids`...) è stato modificato. + +Se inizializzavi i modelli usando parole chiave per gli argomenti, ad esempio `model(inputs_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)`, questo non dovrebbe causare alcun cambiamento. + +Se inizializzavi i modelli con input posizionali per gli argomenti, ad esempio `model(inputs_ids, attention_mask, token_type_ids)`, potrebbe essere necessario ricontrollare l'ordine esatto degli argomenti di input. + +## Migrazione da pytorch-pretrained-bert + +Ecco un breve riepilogo di ciò a cui prestare attenzione durante la migrazione da `pytorch-pretrained-bert` a 🤗 Transformers + +### I modelli restituiscono sempre `tuple` + +La principale modifica di rilievo durante la migrazione da `pytorch-pretrained-bert` a 🤗 Transformers è che il metodo dei modelli di previsione dà sempre una `tupla` con vari elementi a seconda del modello e dei parametri di configurazione. + +Il contenuto esatto delle tuple per ciascun modello è mostrato in dettaglio nelle docstring dei modelli e nella [documentazione](https://huggingface.co/transformers/). + +In quasi tutti i casi, andrà bene prendendo il primo elemento dell'output come quello che avresti precedentemente utilizzato in `pytorch-pretrained-bert`. + +Ecco un esempio di conversione da `pytorch-pretrained-bert` + a 🤗 Transformers per un modello di classificazione `BertForSequenceClassification`: + +```python +# Carichiamo il nostro modello +model = BertForSequenceClassification.from_pretrained("bert-base-uncased") + +# Se usavi questa riga in pytorch-pretrained-bert : +loss = model(input_ids, labels=labels) + +# Ora usa questa riga in 🤗 Transformers per estrarre la perdita dalla tupla di output: +outputs = model(input_ids, labels=labels) +loss = outputs[0] + +# In 🤗 Transformers puoi anche avere accesso ai logit: +loss, logits = outputs[:2] + +# Ed anche agli attention weight se configuri il modello per restituirli (e anche altri output, vedi le docstring e la documentazione) +model = BertForSequenceClassification.from_pretrained(" bert-base-uncased", output_attentions=True) +outputs = model(input_ids, labels=labels) +loss, logits, attentions = outputs +``` + +### Serializzazione + +Modifica sostanziale nel metodo `from_pretrained()`: + +1. I modelli sono ora impostati in modalità di valutazione in maniera predefinita quando usi il metodo `from_pretrained()`. Per addestrarli non dimenticare di riportarli in modalità di addestramento (`model.train()`) per attivare i moduli di dropout. + +2. Gli argomenti aggiuntivi `*inputs` e `**kwargs` forniti al metodo `from_pretrained()` venivano passati direttamente al metodo `__init__()` della classe sottostante del modello. Ora sono usati per aggiornare prima l'attributo di configurazione del modello, che può non funzionare con le classi del modello derivate costruite basandosi sui precedenti esempi di `BertForSequenceClassification`. Più precisamente, gli argomenti posizionali `*inputs` forniti a `from_pretrained()` vengono inoltrati direttamente al metodo `__init__()` del modello mentre gli argomenti keyword `**kwargs` (i) che corrispondono agli attributi della classe di configurazione, vengono utilizzati per aggiornare tali attributi (ii) che non corrispondono ad alcun attributo della classe di configurazione, vengono inoltrati al metodo `__init__()`. + +Inoltre, sebbene non si tratti di una modifica sostanziale, i metodi di serializzazione sono stati standardizzati e probabilmente dovresti passare al nuovo metodo `save_pretrained(save_directory)` se prima usavi qualsiasi altro metodo di serializzazione. + +Ecco un esempio: + +```python +### Carichiamo un modello e un tokenizer +model = BertForSequenceClassification.from_pretrained("bert-base-uncased") +tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + +### Facciamo fare alcune cose al nostro modello e tokenizer +# Es: aggiungiamo nuovi token al vocabolario e agli embending del nostro modello +tokenizer.add_tokens(["[SPECIAL_TOKEN_1]", "[SPECIAL_TOKEN_2]"]) +model.resize_token_embeddings(len(tokenizer)) +# Alleniamo il nostro modello +train(model) + +### Ora salviamo il nostro modello e il tokenizer in una cartella +model.save_pretrained("./my_saved_model_directory/") +tokenizer.save_pretrained("./my_saved_model_directory/") + +### Ricarichiamo il modello e il tokenizer +model = BertForSequenceClassification.from_pretrained("./my_saved_model_directory/") +tokenizer = BertTokenizer.from_pretrained("./my_saved_model_directory/") +``` + +### Ottimizzatori: BertAdam e OpenAIAdam ora sono AdamW, lo scheduling è quello standard PyTorch + +I due ottimizzatori precedenti inclusi, `BertAdam` e `OpenAIAdam`, sono stati sostituiti da un singolo `AdamW` che presenta alcune differenze: + +- implementa solo la correzione del weights decay, +- lo scheduling ora è esterno (vedi sotto), +- anche il gradient clipping ora è esterno (vedi sotto). + +Il nuovo ottimizzatore `AdamW` corrisponde alle API di `Adam` di PyTorch e ti consente di utilizzare metodi PyTorch o apex per lo scheduling e il clipping. + +Lo scheduling è ora standard [PyTorch learning rate schedulers](https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate) e non fanno più parte dell'ottimizzatore. + +Ecco un esempio di linear warmup e decay con `BertAdam` e con `AdamW`: + +```python +# Parametri: +lr = 1e-3 +max_grad_norm = 1.0 +num_training_steps = 1000 +num_warmup_steps = 100 +warmup_proportion = float( num_warmup_steps) / float(num_training_steps) # 0.1 + +### In precedenza l'ottimizzatore BertAdam veniva istanziato in questo modo: +optimizer = BertAdam( + model.parameters(), + lr=lr, + schedule="warmup_linear", + warmup=warmup_proportion, + num_training_steps=num_training_steps, +) +### e usato in questo modo: +for batch in train_data: + loss = model(batch) + loss.backward() + optimizer.step() + +### In 🤗 Transformers, ottimizzatore e schedule sono divisi e usati in questo modo: +optimizer = AdamW( + model.parameters(), lr=lr, correct_bias=False +) # Per riprodurre il comportamento specifico di BertAdam impostare correct_bias=False +scheduler = get_linear_schedule_with_warmup( + optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps +) # PyTorch scheduler +### e va usato così: +for batch in train_data: + loss = model(batch) + loss.backward() + torch.nn.utils.clip_grad_norm_( + model.parameters(), max_grad_norm + ) # Gradient clipping non è più in AdamW (quindi puoi usare amp senza problemi) + optimizer.step() + scheduler.step() +``` From 5e10dfefc62c11a8f25aeabfacb62e132b584036 Mon Sep 17 00:00:00 2001 From: Jason Phang Date: Thu, 16 Mar 2023 09:00:53 -0400 Subject: [PATCH 111/277] LLaMA Implementation (#21955) * LLaMA * sharding and docs * tweak * black * inits * ruff * LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP * init * no checkpoint * docs * ruff * type_vocab_size * tokenizer fixes * tokenizer fixes * Update tokenization_llama.py * Update tokenization_llama.py * Update configuration_llama.py * Update modeling_llama.py * tokenizer add_bos by default * licenses * remove decoder * norms and mlp * rope overhaul * tweaks * black * mention OPT implementation * off-by-one naming * typo * fix * tokenization fix and slicing bug * padding config * cleanup * black * update tests * undo typo * fix vocab caching logic * ruff * docbuilder * attn fix from BlackSamorez * initial feedback * typo * docs * llama case * llama case * load checkpoint docs * comment about tokenizer * tokenizer defaults * clear past_key_values if use_cache=False * last tweaks * last tweaks * last tweaks * last tweaks --------- Co-authored-by: Stella Biderman --- README.md | 1 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/en/_toctree.yml | 2 + docs/source/en/index.mdx | 2 + docs/source/en/model_doc/llama.mdx | 66 ++ docs/source/en/tasks/language_modeling.mdx | 2 +- src/transformers/__init__.py | 18 +- src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + src/transformers/models/auto/modeling_auto.py | 2 + .../models/auto/tokenization_auto.py | 1 + src/transformers/models/llama/__init__.py | 76 ++ .../models/llama/configuration_llama.py | 112 +++ .../llama/convert_llama_weights_to_hf.py | 282 ++++++ .../models/llama/modeling_llama.py | 833 ++++++++++++++++++ .../models/llama/tokenization_llama.py | 231 +++++ src/transformers/utils/dummy_pt_objects.py | 21 + .../utils/dummy_sentencepiece_objects.py | 7 + tests/models/llama/__init__.py | 0 tests/models/llama/test_modeling_llama.py | 304 +++++++ utils/check_config_docstrings.py | 1 + utils/check_repo.py | 1 + 27 files changed, 1970 insertions(+), 2 deletions(-) create mode 100644 docs/source/en/model_doc/llama.mdx create mode 100644 src/transformers/models/llama/__init__.py create mode 100644 src/transformers/models/llama/configuration_llama.py create mode 100644 src/transformers/models/llama/convert_llama_weights_to_hf.py create mode 100755 src/transformers/models/llama/modeling_llama.py create mode 100644 src/transformers/models/llama/tokenization_llama.py create mode 100644 tests/models/llama/__init__.py create mode 100644 tests/models/llama/test_modeling_llama.py diff --git a/README.md b/README.md index ff6da10fdf3f..6397d14adaad 100644 --- a/README.md +++ b/README.md @@ -364,6 +364,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[LED](https://huggingface.co/docs/transformers/model_doc/led)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LeViT](https://huggingface.co/docs/transformers/model_doc/levit)** (from Meta AI) released with the paper [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https://arxiv.org/abs/2104.01136) by Ben Graham, Alaaeldin El-Nouby, Hugo Touvron, Pierre Stock, Armand Joulin, Hervé Jégou, Matthijs Douze. 1. **[LiLT](https://huggingface.co/docs/transformers/model_doc/lilt)** (from South China University of Technology) released with the paper [LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding](https://arxiv.org/abs/2202.13669) by Jiapeng Wang, Lianwen Jin, Kai Ding. +1. **[LLaMA](https://huggingface.co/docs/transformers/main/model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. 1. **[Longformer](https://huggingface.co/docs/transformers/model_doc/longformer)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang. 1. **[LUKE](https://huggingface.co/docs/transformers/model_doc/luke)** (from Studio Ousia) released with the paper [LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention](https://arxiv.org/abs/2010.01057) by Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto. diff --git a/README_es.md b/README_es.md index 57b6272d4d17..899140210cf1 100644 --- a/README_es.md +++ b/README_es.md @@ -352,6 +352,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[LED](https://huggingface.co/docs/transformers/model_doc/led)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LeViT](https://huggingface.co/docs/transformers/model_doc/levit)** (from Meta AI) released with the paper [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https://arxiv.org/abs/2104.01136) by Ben Graham, Alaaeldin El-Nouby, Hugo Touvron, Pierre Stock, Armand Joulin, Hervé Jégou, Matthijs Douze. 1. **[LiLT](https://huggingface.co/docs/transformers/model_doc/lilt)** (from South China University of Technology) released with the paper [LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding](https://arxiv.org/abs/2202.13669) by Jiapeng Wang, Lianwen Jin, Kai Ding. +1. **[LLaMA](https://huggingface.co/docs/transformers/main/model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. 1. **[Longformer](https://huggingface.co/docs/transformers/model_doc/longformer)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang. 1. **[LUKE](https://huggingface.co/docs/transformers/model_doc/luke)** (from Studio Ousia) released with the paper [LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention](https://arxiv.org/abs/2010.01057) by Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto. diff --git a/README_hd.md b/README_hd.md index 2355369ad8d3..826306cf67bf 100644 --- a/README_hd.md +++ b/README_hd.md @@ -324,6 +324,7 @@ conda install -c huggingface transformers 1. **[LED](https://huggingface.co/docs/transformers/model_doc/led)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LeViT](https://huggingface.co/docs/transformers/model_doc/levit)** (मेटा AI से) साथ वाला पेपर [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https:/ /arxiv.org/abs/2104.01136) बेन ग्राहम, अलाएल्डिन एल-नौबी, ह्यूगो टौवरन, पियरे स्टॉक, आर्मंड जौलिन, हर्वे जेगौ, मैथिज डूज़ द्वारा। 1. **[LiLT](https://huggingface.co/docs/transformers/model_doc/lilt)** (दक्षिण चीन प्रौद्योगिकी विश्वविद्यालय से) साथ में कागज [LiLT: एक सरल लेकिन प्रभावी भाषा-स्वतंत्र लेआउट ट्रांसफार्मर संरचित दस्तावेज़ समझ के लिए](https://arxiv.org/abs/2202.13669) जियापेंग वांग, लियानवेन जिन, काई डिंग द्वारा पोस्ट किया गया। +1. **[LLaMA](https://huggingface.co/docs/transformers/main/model_doc/llama)** (The FAIR team of Meta AI से) Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. द्वाराअनुसंधान पत्र [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) के साथ जारी किया गया 1. **[Longformer](https://huggingface.co/docs/transformers/model_doc/longformer)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (मैंडी गुओ, जोशुआ आइंस्ली, डेविड यूथस, सैंटियागो ओंटानन, जियानमो नि, यूं-हुआन सुंग, यिनफेई यांग द्वारा पोस्ट किया गया। 1. **[LUKE](https://huggingface.co/docs/transformers/model_doc/luke)** (स्टूडियो औसिया से) साथ में पेपर [LUKE: डीप कॉन्टेक्स्टुअलाइज्ड एंटिटी रिप्रेजेंटेशन विद एंटिटी-अवेयर सेल्फ-अटेंशन](https ://arxiv.org/abs/2010.01057) Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto द्वारा। diff --git a/README_ja.md b/README_ja.md index 9d4aceed01df..b45cc68ea6b2 100644 --- a/README_ja.md +++ b/README_ja.md @@ -386,6 +386,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[LED](https://huggingface.co/docs/transformers/model_doc/led)** (AllenAI から) Iz Beltagy, Matthew E. Peters, Arman Cohan から公開された研究論文: [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) 1. **[LeViT](https://huggingface.co/docs/transformers/model_doc/levit)** (Meta AI から) Ben Graham, Alaaeldin El-Nouby, Hugo Touvron, Pierre Stock, Armand Joulin, Hervé Jégou, Matthijs Douze から公開された研究論文: [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https://arxiv.org/abs/2104.01136) 1. **[LiLT](https://huggingface.co/docs/transformers/model_doc/lilt)** (South China University of Technology から) Jiapeng Wang, Lianwen Jin, Kai Ding から公開された研究論文: [LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding](https://arxiv.org/abs/2202.13669) +1. **[LLaMA](https://huggingface.co/docs/transformers/main/model_doc/llama)** (The FAIR team of Meta AI から) Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. から公開された研究論文 [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) 1. **[Longformer](https://huggingface.co/docs/transformers/model_doc/longformer)** (AllenAI から) Iz Beltagy, Matthew E. Peters, Arman Cohan から公開された研究論文: [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) 1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (Google AI から) Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang から公開された研究論文: [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) 1. **[LUKE](https://huggingface.co/docs/transformers/model_doc/luke)** (Studio Ousia から) Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto から公開された研究論文: [LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention](https://arxiv.org/abs/2010.01057) diff --git a/README_ko.md b/README_ko.md index b8e69fccfa6b..a5c0b8cf1eee 100644 --- a/README_ko.md +++ b/README_ko.md @@ -301,6 +301,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[LED](https://huggingface.co/docs/transformers/model_doc/led)** (AllenAI 에서) Iz Beltagy, Matthew E. Peters, Arman Cohan 의 [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) 논문과 함께 발표했습니다. 1. **[LeViT](https://huggingface.co/docs/transformers/model_doc/levit)** (Meta AI 에서) Ben Graham, Alaaeldin El-Nouby, Hugo Touvron, Pierre Stock, Armand Joulin, Hervé Jégou, Matthijs Douze 의 [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https://arxiv.org/abs/2104.01136) 논문과 함께 발표했습니다. 1. **[LiLT](https://huggingface.co/docs/transformers/model_doc/lilt)** (South China University of Technology 에서) Jiapeng Wang, Lianwen Jin, Kai Ding 의 [LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding](https://arxiv.org/abs/2202.13669) 논문과 함께 발표했습니다. +1. **[LLaMA](https://huggingface.co/docs/transformers/main/model_doc/llama)** (The FAIR team of Meta AI 에서 제공)은 Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample.의 [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971)논문과 함께 발표했습니다. 1. **[Longformer](https://huggingface.co/docs/transformers/model_doc/longformer)** (AllenAI 에서) Iz Beltagy, Matthew E. Peters, Arman Cohan 의 [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) 논문과 함께 발표했습니다. 1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (Google AI 에서) Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang 의 [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) 논문과 함께 발표했습니다. 1. **[LUKE](https://huggingface.co/docs/transformers/model_doc/luke)** (Studio Ousia 에서) Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto 의 [LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention](https://arxiv.org/abs/2010.01057) 논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index 8527a522949b..9ae3bc24494f 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -325,6 +325,7 @@ conda install -c huggingface transformers 1. **[LED](https://huggingface.co/docs/transformers/model_doc/led)** (来自 AllenAI) 伴随论文 [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) 由 Iz Beltagy, Matthew E. Peters, Arman Cohan 发布。 1. **[LeViT](https://huggingface.co/docs/transformers/model_doc/levit)** (来自 Meta AI) 伴随论文 [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https://arxiv.org/abs/2104.01136) 由 Ben Graham, Alaaeldin El-Nouby, Hugo Touvron, Pierre Stock, Armand Joulin, Hervé Jégou, Matthijs Douze 发布。 1. **[LiLT](https://huggingface.co/docs/transformers/model_doc/lilt)** (来自 South China University of Technology) 伴随论文 [LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding](https://arxiv.org/abs/2202.13669) 由 Jiapeng Wang, Lianwen Jin, Kai Ding 发布。 +1. **[LLaMA](https://huggingface.co/docs/transformers/main/model_doc/llama)** (来自 The FAIR team of Meta AI) 伴随论文 [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) 由 Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample 发布。 1. **[Longformer](https://huggingface.co/docs/transformers/model_doc/longformer)** (来自 AllenAI) 伴随论文 [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) 由 Iz Beltagy, Matthew E. Peters, Arman Cohan 发布。 1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (来自 Google AI) released 伴随论文 [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) 由 Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang 发布。 1. **[LUKE](https://huggingface.co/docs/transformers/model_doc/luke)** (来自 Studio Ousia) 伴随论文 [LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention](https://arxiv.org/abs/2010.01057) 由 Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index f5b087f412e1..53847bf6739a 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -337,6 +337,7 @@ conda install -c huggingface transformers 1. **[LED](https://huggingface.co/docs/transformers/model_doc/led)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LeViT](https://huggingface.co/docs/transformers/model_doc/levit)** (from Meta AI) released with the paper [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https://arxiv.org/abs/2104.01136) by Ben Graham, Alaaeldin El-Nouby, Hugo Touvron, Pierre Stock, Armand Joulin, Hervé Jégou, Matthijs Douze. 1. **[LiLT](https://huggingface.co/docs/transformers/model_doc/lilt)** (from South China University of Technology) released with the paper [LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding](https://arxiv.org/abs/2202.13669) by Jiapeng Wang, Lianwen Jin, Kai Ding. +1. **[LLaMA](https://huggingface.co/docs/transformers/main/model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. 1. **[Longformer](https://huggingface.co/docs/transformers/model_doc/longformer)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang. 1. **[LUKE](https://huggingface.co/docs/transformers/model_doc/luke)** (from Studio Ousia) released with the paper [LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention](https://arxiv.org/abs/2010.01057) by Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto. diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 947a40978294..d8833f3efe08 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -319,6 +319,8 @@ title: Jukebox - local: model_doc/led title: LED + - local: model_doc/llama + title: LLaMA - local: model_doc/longformer title: Longformer - local: model_doc/longt5 diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 1f4f5a0d43f6..ea1ab27e7970 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -138,6 +138,7 @@ The documentation is organized into five sections: 1. **[LED](model_doc/led)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LeViT](model_doc/levit)** (from Meta AI) released with the paper [LeViT: A Vision Transformer in ConvNet's Clothing for Faster Inference](https://arxiv.org/abs/2104.01136) by Ben Graham, Alaaeldin El-Nouby, Hugo Touvron, Pierre Stock, Armand Joulin, Hervé Jégou, Matthijs Douze. 1. **[LiLT](model_doc/lilt)** (from South China University of Technology) released with the paper [LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding](https://arxiv.org/abs/2202.13669) by Jiapeng Wang, Lianwen Jin, Kai Ding. +1. **[LLaMA](model_doc/llama)** (from The FAIR team of Meta AI) released with the paper [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. 1. **[Longformer](model_doc/longformer)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan. 1. **[LongT5](model_doc/longt5)** (from Google AI) released with the paper [LongT5: Efficient Text-To-Text Transformer for Long Sequences](https://arxiv.org/abs/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo Ni, Yun-Hsuan Sung, Yinfei Yang. 1. **[LUKE](model_doc/luke)** (from Studio Ousia) released with the paper [LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention](https://arxiv.org/abs/2010.01057) by Ikuya Yamada, Akari Asai, Hiroyuki Shindo, Hideaki Takeda, Yuji Matsumoto. @@ -330,6 +331,7 @@ Flax), PyTorch, and/or TensorFlow. | LED | ✅ | ✅ | ✅ | ✅ | ❌ | | LeViT | ❌ | ❌ | ✅ | ❌ | ❌ | | LiLT | ❌ | ❌ | ✅ | ❌ | ❌ | +| LLaMA | ✅ | ❌ | ✅ | ❌ | ❌ | | Longformer | ✅ | ✅ | ✅ | ✅ | ❌ | | LongT5 | ❌ | ❌ | ✅ | ❌ | ✅ | | LUKE | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/model_doc/llama.mdx b/docs/source/en/model_doc/llama.mdx new file mode 100644 index 000000000000..3f6ea3409bb8 --- /dev/null +++ b/docs/source/en/model_doc/llama.mdx @@ -0,0 +1,66 @@ + + +# LLaMA + +## Overview + +The LLaMA model was proposed in [LLaMA: Open and Efficient Foundation Language Models](LLaMA: Open and Efficient Foundation Language Models) by Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, Guillaume Lample. It is a collection of foundation language models ranging from 7B to 65B parameters. + +The abstract from the paper is the following: + +*We introduce LLaMA, a collection of foundation language models ranging from 7B to 65B parameters. We train our models on trillions of tokens, and show that it is possible to train state-of-the-art models using publicly available datasets exclusively, without resorting to proprietary and inaccessible datasets. In particular, LLaMA-13B outperforms GPT-3 (175B) on most benchmarks, and LLaMA-65B is competitive with the best models, Chinchilla-70B and PaLM-540B. We release all our models to the research community. * + +Tips: + +- Weights for the LLaMA models can be obtained from by filling out [this form]()https://docs.google.com/forms/d/e/1FAIpQLSfqNECQnMkycAp2jP4Z9TFX0cGR4uf7b_fBxjY_OjhJILlKGA/viewform?usp=send_form) +- After downloading the weights, they will need to be converted to the Hugging Face Transformers format using the [conversion script](/src/transformers/models/llama/convert_llama_weights_to_hf.py). The script can be called with the following (example) command: + +```bash +python src/transformers/models/llama/convert_llama_weights_to_hf.py \ + --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path +``` + +- After conversion, the model and tokenizer can be loaded via: + +```python +tokenizer = transformers.LlamaTokenizer.from_pretrained("/output/path/tokenizer/") +model = transformers.LlamaForCausalLM.from_pretrained("/output/path/llama-7b/") +``` + +- The LLaMA tokenizer is based on [sentencepiece](https://github.com/google/sentencepiece). One quick of sentencepiece is that when decoding a sequence, if the first token is the start of the word (e.g. "Banana"), the tokenizer does not prepend the prefix space to the string. To have the tokenizer output the prefix space, set `decode_with_prefix_space=True` in the `LlamaTokenizer` object or in the tokenizer configuration. + +This model was contributed by [zphang](https://huggingface.co/zphang) with contributions from [BlackSamorez](https://huggingface.co/BlackSamorez). The code of the implementation in Hugging Face is based on GPT-NeoX [here](https://github.com/EleutherAI/gpt-neox). The original code of the authors can be found [here](https://github.com/facebookresearch/llama). + +## LlamaConfig + +[[autodoc]] LlamaConfig + + +## LlamaTokenizer + +[[autodoc]] LlamaTokenizer + - build_inputs_with_special_tokens + - get_special_tokens_mask + - create_token_type_ids_from_sequences + - save_vocabulary + +## LlamaModel + +[[autodoc]] LlamaModel + - forward + + +## LlamaForCausalLM + +[[autodoc]] LlamaForCausalLM + - forward diff --git a/docs/source/en/tasks/language_modeling.mdx b/docs/source/en/tasks/language_modeling.mdx index 3412c642b30f..d8801888d378 100644 --- a/docs/source/en/tasks/language_modeling.mdx +++ b/docs/source/en/tasks/language_modeling.mdx @@ -34,7 +34,7 @@ Choose one of the following architectures: -[BART](../model_doc/bart), [BERT](../model_doc/bert), [Bert Generation](../model_doc/bert-generation), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BioGpt](../model_doc/biogpt), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CodeGen](../model_doc/codegen), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [GIT](../model_doc/git), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT NeoX](../model_doc/gpt_neox), [GPT NeoX Japanese](../model_doc/gpt_neox_japanese), [GPT-J](../model_doc/gptj), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MVP](../model_doc/mvp), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Pegasus](../model_doc/pegasus), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [Speech2Text2](../model_doc/speech_to_text_2), [Transformer-XL](../model_doc/transfo-xl), [TrOCR](../model_doc/trocr), [XGLM](../model_doc/xglm), [XLM](../model_doc/xlm), [XLM-ProphetNet](../model_doc/xlm-prophetnet), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod) +[BART](../model_doc/bart), [BERT](../model_doc/bert), [Bert Generation](../model_doc/bert-generation), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BioGpt](../model_doc/biogpt), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CodeGen](../model_doc/codegen), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [GIT](../model_doc/git), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT NeoX](../model_doc/gpt_neox), [GPT NeoX Japanese](../model_doc/gpt_neox_japanese), [GPT-J](../model_doc/gptj), [LLaMA](../model_doc/llama), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MVP](../model_doc/mvp), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Pegasus](../model_doc/pegasus), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [Speech2Text2](../model_doc/speech_to_text_2), [Transformer-XL](../model_doc/transfo-xl), [TrOCR](../model_doc/trocr), [XGLM](../model_doc/xglm), [XLM](../model_doc/xlm), [XLM-ProphetNet](../model_doc/xlm-prophetnet), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index f0add2263abc..1bc0747ac7f1 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -348,6 +348,7 @@ "models.led": ["LED_PRETRAINED_CONFIG_ARCHIVE_MAP", "LEDConfig", "LEDTokenizer"], "models.levit": ["LEVIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LevitConfig"], "models.lilt": ["LILT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LiltConfig"], + "models.llama": ["LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP", "LlamaConfig"], "models.longformer": ["LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "LongformerConfig", "LongformerTokenizer"], "models.longt5": ["LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP", "LongT5Config"], "models.luke": ["LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP", "LukeConfig", "LukeTokenizer"], @@ -667,6 +668,7 @@ _import_structure["models.fnet"].append("FNetTokenizer") _import_structure["models.gpt_sw3"].append("GPTSw3Tokenizer") _import_structure["models.layoutxlm"].append("LayoutXLMTokenizer") + _import_structure["models.llama"].append("LlamaTokenizer") _import_structure["models.m2m_100"].append("M2M100Tokenizer") _import_structure["models.marian"].append("MarianTokenizer") _import_structure["models.mbart"].append("MBartTokenizer") @@ -1798,6 +1800,13 @@ "LiltPreTrainedModel", ] ) + _import_structure["models.llama"].extend( + [ + "LlamaForCausalLM", + "LlamaModel", + "LlamaPreTrainedModel", + ] + ) _import_structure["models.longformer"].extend( [ "LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -3965,6 +3974,7 @@ from .models.led import LED_PRETRAINED_CONFIG_ARCHIVE_MAP, LEDConfig, LEDTokenizer from .models.levit import LEVIT_PRETRAINED_CONFIG_ARCHIVE_MAP, LevitConfig from .models.lilt import LILT_PRETRAINED_CONFIG_ARCHIVE_MAP, LiltConfig + from .models.llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig from .models.longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig, LongformerTokenizer from .models.longt5 import LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP, LongT5Config from .models.luke import LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP, LukeConfig, LukeTokenizer @@ -4258,6 +4268,7 @@ from .models.fnet import FNetTokenizer from .models.gpt_sw3 import GPTSw3Tokenizer from .models.layoutxlm import LayoutXLMTokenizer + from .models.llama import LlamaTokenizer from .models.m2m_100 import M2M100Tokenizer from .models.marian import MarianTokenizer from .models.mbart import MBart50Tokenizer, MBartTokenizer @@ -4476,9 +4487,9 @@ TypicalLogitsWarper, top_k_top_p_filtering, ) - from .modeling_utils import PreTrainedModel # PyTorch model imports + from .modeling_utils import PreTrainedModel from .models.albert import ( ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, AlbertForMaskedLM, @@ -5188,6 +5199,11 @@ LiltModel, LiltPreTrainedModel, ) + from .models.llama import ( + LlamaForCausalLM, + LlamaModel, + LlamaPreTrainedModel, + ) from .models.longformer import ( LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST, LongformerForMaskedLM, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 9951da553807..d366eefdcbb0 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -103,6 +103,7 @@ led, levit, lilt, + llama, longformer, longt5, luke, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 70ab1bc9b10c..9a8ef79c7c11 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -109,6 +109,7 @@ ("led", "LEDConfig"), ("levit", "LevitConfig"), ("lilt", "LiltConfig"), + ("llama", "LlamaConfig"), ("longformer", "LongformerConfig"), ("longt5", "LongT5Config"), ("luke", "LukeConfig"), @@ -286,6 +287,7 @@ ("led", "LED_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("levit", "LEVIT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("lilt", "LILT_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("llama", "LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("longformer", "LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("longt5", "LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("luke", "LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -466,6 +468,7 @@ ("led", "LED"), ("levit", "LeViT"), ("lilt", "LiLT"), + ("llama", "LLaMA"), ("longformer", "Longformer"), ("longt5", "LongT5"), ("luke", "LUKE"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index c29965944aaa..6da804733a5f 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -107,6 +107,7 @@ ("led", "LEDModel"), ("levit", "LevitModel"), ("lilt", "LiltModel"), + ("llama", "LlamaModel"), ("longformer", "LongformerModel"), ("longt5", "LongT5Model"), ("luke", "LukeModel"), @@ -359,6 +360,7 @@ ("gpt_neox", "GPTNeoXForCausalLM"), ("gpt_neox_japanese", "GPTNeoXJapaneseForCausalLM"), ("gptj", "GPTJForCausalLM"), + ("llama", "LlamaForCausalLM"), ("marian", "MarianForCausalLM"), ("mbart", "MBartForCausalLM"), ("megatron-bert", "MegatronBertForCausalLM"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 3f81c5e1f935..2e6ddc0b132c 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -167,6 +167,7 @@ ("layoutxlm", ("LayoutXLMTokenizer", "LayoutXLMTokenizerFast" if is_tokenizers_available() else None)), ("led", ("LEDTokenizer", "LEDTokenizerFast" if is_tokenizers_available() else None)), ("lilt", ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast" if is_tokenizers_available() else None)), + ("llama", ("LlamaTokenizer" if is_sentencepiece_available() else None, None)), ("longformer", ("LongformerTokenizer", "LongformerTokenizerFast" if is_tokenizers_available() else None)), ( "longt5", diff --git a/src/transformers/models/llama/__init__.py b/src/transformers/models/llama/__init__.py new file mode 100644 index 000000000000..2501c282f0ba --- /dev/null +++ b/src/transformers/models/llama/__init__.py @@ -0,0 +1,76 @@ +# Copyright 2022 EleutherAI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import ( + OptionalDependencyNotAvailable, + _LazyModule, + is_sentencepiece_available, + is_torch_available, +) + + +_import_structure = { + "configuration_llama": ["LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP", "LlamaConfig"], +} + +try: + if not is_sentencepiece_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["tokenization_llama"] = ["LlamaTokenizer"] + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_llama"] = [ + "LlamaForCausalLM", + "LlamaModel", + "LlamaPreTrainedModel", + ] + + +if TYPE_CHECKING: + from .configuration_llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig + + try: + if not is_sentencepiece_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .tokenization_llama import LlamaTokenizer + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_llama import ( + LlamaForCausalLM, + LlamaModel, + LlamaPreTrainedModel, + ) + + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/llama/configuration_llama.py b/src/transformers/models/llama/configuration_llama.py new file mode 100644 index 000000000000..5421d429b101 --- /dev/null +++ b/src/transformers/models/llama/configuration_llama.py @@ -0,0 +1,112 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" LLaMA model configuration""" + +from ...configuration_utils import PretrainedConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP = {} + + +class LlamaConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`~LlamaModel`]. It is used to instantiate an LLaMA + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the LLaMA-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`~LlamaModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings(`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + Example: + + ```python + >>> from transformers import LlamaModel, LlamaConfig + + >>> # Initializing a LLaMA llama-7b style configuration + >>> configuration = LlamaConfig() + + >>> # Initializing a model from the llama-7b style configuration + >>> model = LlamaModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "llama" + + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + hidden_act="silu", + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) diff --git a/src/transformers/models/llama/convert_llama_weights_to_hf.py b/src/transformers/models/llama/convert_llama_weights_to_hf.py new file mode 100644 index 000000000000..521ea3dbb660 --- /dev/null +++ b/src/transformers/models/llama/convert_llama_weights_to_hf.py @@ -0,0 +1,282 @@ +# Copyright 2022 EleutherAI and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import math +import os +import shutil + +import torch + + +""" +Sample usage: + + ``` + python src/transformers/models/llama/convert_llama_weights_to_hf.py \ + --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path + ``` + +Thereafter, models can be loaded via: + + ``` + tokenizer = transformers.LlamaTokenizer.from_pretrained("/output/path/tokenizer/") + + model = transformers.LlamaForCausalLM.from_pretrained("/output/path/llama-7b/") + ``` +""" + +INTERMEDIATE_SIZE_MAP = { + "7B": 11008, + "13B": 13824, + "30B": 17920, + "65B": 22016, +} +NUM_SHARDS = { + "7B": 1, + "13B": 2, + "30B": 4, + "65B": 8, +} + + +def compute_intermediate_size(n): + return int(math.ceil(n * 8 / 3) + 255) // 256 * 256 + + +def read_json(path): + with open(path, "r") as f: + return json.load(f) + + +def write_json(text, path): + with open(path, "w") as f: + json.dump(text, f) + + +def write_model(model_path, input_base_path, model_size): + assert model_size in NUM_SHARDS + os.makedirs(model_path, exist_ok=True) + + params = read_json(os.path.join(input_base_path, "params.json")) + num_shards = NUM_SHARDS[model_size] + n_layers = params["n_layers"] + n_heads = params["n_heads"] + n_heads_per_shard = n_heads // num_shards + dim = params["dim"] + dims_per_head = dim // n_heads + base = 10000.0 + inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head)) + + # permute for sliced rotary + def permute(w): + return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim) + + # Load weights + if model_size == "7B": + # Not shared + # (The sharded implementation would also work, but this is simpler.) + loaded = torch.load(os.path.join(input_base_path, "consolidated.00.pth"), map_location="cpu") + else: + # Sharded + loaded = [ + torch.load(os.path.join(input_base_path, f"consolidated.{i:02d}.pth"), map_location="cpu") + for i in range(num_shards) + ] + param_count = 0 + index_dict = {"weight_map": {}} + for layer_i in range(n_layers): + filename = "pytorch_model-{:05d}-of-{:05d}.bin".format( + layer_i + 1, + n_layers + 1, + ) + if model_size == "7B": + # Unsharded + state_dict = { + f"model.layers.{layer_i}.self_attn.q_proj.weight": permute( + loaded[f"layers.{layer_i}.attention.wq.weight"] + ), + f"model.layers.{layer_i}.self_attn.k_proj.weight": permute( + loaded[f"layers.{layer_i}.attention.wk.weight"] + ), + f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"], + f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"], + f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"], + f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"], + f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"], + f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"], + f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"], + } + else: + # Sharded + state_dict = { + f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][f"layers.{layer_i}.attention_norm.weight"], + f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][ + f"layers.{layer_i}.ffn_norm.weight" + ], + } + state_dict[f"model.layers.{layer_i}.self_attn.q_proj.weight"] = permute( + torch.cat( + [ + loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(n_heads_per_shard, dims_per_head, dim) + for i in range(num_shards) + ], + dim=0, + ).reshape(dim, dim) + ) + state_dict[f"model.layers.{layer_i}.self_attn.k_proj.weight"] = permute( + torch.cat( + [ + loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(n_heads_per_shard, dims_per_head, dim) + for i in range(num_shards) + ], + dim=0, + ).reshape(dim, dim) + ) + state_dict[f"model.layers.{layer_i}.self_attn.v_proj.weight"] = torch.cat( + [ + loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(n_heads_per_shard, dims_per_head, dim) + for i in range(num_shards) + ], + dim=0, + ).reshape(dim, dim) + + state_dict[f"model.layers.{layer_i}.self_attn.o_proj.weight"] = torch.cat( + [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(num_shards)], dim=1 + ) + state_dict[f"model.layers.{layer_i}.mlp.gate_proj.weight"] = torch.cat( + [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(num_shards)], dim=0 + ) + state_dict[f"model.layers.{layer_i}.mlp.down_proj.weight"] = torch.cat( + [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(num_shards)], dim=1 + ) + state_dict[f"model.layers.{layer_i}.mlp.up_proj.weight"] = torch.cat( + [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(num_shards)], dim=0 + ) + + state_dict[f"model.layers.{layer_i}.self_attn.rotary_emb.inv_freq"] = inv_freq + for k, v in state_dict.items(): + index_dict["weight_map"][k] = filename + param_count += v.numel() + torch.save(state_dict, os.path.join(model_path, filename)) + + filename = "pytorch_model-{:05d}-of-{:05d}.bin".format( + n_layers + 1, + n_layers + 1, + ) + if model_size == "7B": + # Unsharded + state_dict = { + "model.embed_tokens.weight": loaded["tok_embeddings.weight"], + "model.norm.weight": loaded["norm.weight"], + "lm_head.weight": loaded["output.weight"], + } + else: + state_dict = { + "model.norm.weight": loaded[0]["norm.weight"], + "model.embed_tokens.weight": torch.cat( + [loaded[i]["tok_embeddings.weight"] for i in range(num_shards)], dim=1 + ), + "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(num_shards)], dim=0), + } + + for k, v in state_dict.items(): + index_dict["weight_map"][k] = filename + param_count += v.numel() + torch.save(state_dict, os.path.join(model_path, filename)) + + # Write configs + index_dict["metadata"] = {"total_size": param_count * 2} + write_json(index_dict, os.path.join(model_path, "pytorch_model.bin.index.json")) + config_out = { + "architectures": ["LlamaForCausalLM"], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": dim, + "intermediate_size": compute_intermediate_size(dim), + "initializer_range": 0.02, + "max_sequence_length": 2048, + "model_type": "llama", + "num_attention_heads": params["n_heads"], + "num_hidden_layers": params["n_layers"], + "pad_token_id": 0, + "rms_norm_eps": params["norm_eps"], + "torch_dtype": "float16", + "transformers_version": "4.27.0.dev0", + "use_cache": True, + "vocab_size": 32000, + } + write_json( + config_out, + os.path.join(model_path, "config.json"), + ) + generation_config = { + "_from_model_config": True, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.27.0.dev0", + } + write_json( + generation_config, + os.path.join(model_path, "generation_config.json"), + ) + + +def write_tokenizer(tokenizer_path, input_tokenizer_path): + os.makedirs(tokenizer_path, exist_ok=True) + write_json({}, os.path.join(tokenizer_path, "special_tokens_map.json")) + write_json( + { + "bos_token": "", + "eos_token": "", + "model_max_length": int(1e30), + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + }, + os.path.join(tokenizer_path, "tokenizer_config.json"), + ) + shutil.copyfile(input_tokenizer_path, os.path.join(tokenizer_path, "tokenizer.model")) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--input_dir", + help="Location of LLaMA weights, which contains tokenizer.model and model folders", + ) + parser.add_argument( + "--model_size", + choices=["7B", "13B", "30B", "65B", "tokenizer_only"], + ) + parser.add_argument( + "--output_dir", + help="Location to write HF model and tokenizer", + ) + args = parser.parse_args() + if args.model_size != "tokenizer_only": + write_model( + model_path=os.path.join(args.output_dir, "llama-{}".format(args.model_size).lower()), + input_base_path=os.path.join(args.input_dir, args.model_size), + model_size=args.model_size, + ) + write_tokenizer( + tokenizer_path=os.path.join(args.output_dir, "tokenizer"), + input_tokenizer_path=os.path.join(args.input_dir, "tokenizer.model"), + ) + + +if __name__ == "__main__": + main() diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py new file mode 100755 index 000000000000..40a3f0f802a0 --- /dev/null +++ b/src/transformers/models/llama/modeling_llama.py @@ -0,0 +1,833 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch LLaMA model.""" +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss + +from ...activations import ACT2FN +from ...modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, +) +from ...modeling_utils import PreTrainedModel +from ...utils import ( + add_start_docstrings, + logging, + replace_return_docstrings, +) +from .configuration_llama import LlamaConfig + + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "LlamaConfig" + + +def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) + mask_cond = torch.arange(mask.size(-1)) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +class LlamaRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states + + +class LlamaRotaryEmbedding(torch.nn.Module): + def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): + super().__init__() + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim)) + self.register_buffer("inv_freq", inv_freq) + + # Build here to make `torch.jit.trace` work. + self.max_seq_len_cached = max_position_embeddings + t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + self.cos_cached = emb.cos()[None, None, :, :] + self.sin_cached = emb.sin()[None, None, :, :] + + def forward(self, x, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case. + if seq_len > self.max_seq_len_cached: + self.max_seq_len_cached = seq_len + t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, self.inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1).to(x.device) + self.cos_cached = emb.cos()[None, None, :, :].to(dtype=x.dtype) + self.sin_cached = emb.sin()[None, None, :, :].to(dtype=x.dtype) + return ( + self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype, device=x.device), + self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype, device=x.device), + ) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, offset: int = 0): + cos = cos[..., offset : q.shape[-2] + offset, :] + sin = sin[..., offset : q.shape[-2] + offset, :] + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class LlamaMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + ): + super().__init__() + self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False) + self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False) + self.act_fn = ACT2FN[hidden_act] + + def forward(self, x): + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + +class LlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + hidden_size: int, + num_heads: int, + ): + super().__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + + if (self.head_dim * num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {num_heads})." + ) + self.q_proj = nn.Linear( + hidden_size, + num_heads * self.head_dim, + bias=False, + ) + self.k_proj = nn.Linear( + hidden_size, + num_heads * self.head_dim, + bias=False, + ) + self.v_proj = nn.Linear( + hidden_size, + num_heads * self.head_dim, + bias=False, + ) + self.o_proj = nn.Linear( + num_heads * self.head_dim, + hidden_size, + bias=False, + ) + self.rotary_emb = LlamaRotaryEmbedding(self.head_dim) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + offset = 0 + if past_key_value is not None: + offset = past_key_value[0].shape[-2] + kv_seq_len += offset + cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, offset=offset) + # [bsz, nh, t, hd] + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, q_len, kv_seq_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights + attention_mask + attn_weights = torch.max(attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)) + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) + + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class LlamaDecoderLayer(nn.Module): + def __init__(self, config: LlamaConfig): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = LlamaAttention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + ) + self.mlp = LlamaMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + ) + self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + past_key_value=past_key_value, + attention_mask=attention_mask, + output_attentions=output_attentions, + use_cache=use_cache, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +LLAMA_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`LlamaConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare OPT Model outputting raw hidden-states without any specific head on top.", + LLAMA_START_DOCSTRING, +) +class LlamaPreTrainedModel(PreTrainedModel): + config_class = LlamaConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["LlamaDecoderLayer"] + _keys_to_ignore_on_load_unexpected = [r"decoder\.version"] + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (LlamaDecoderLayer)): + module.gradient_checkpointing = value + + +LLAMA_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", + LLAMA_START_DOCSTRING, +) +class LlamaModel(LlamaPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] + + Args: + config: LlamaConfig + """ + + def __init__(self, config: LlamaConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length + ).to(inputs_embeds.device) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + seq_length_with_past = seq_length + past_key_values_length = 0 + if past_key_values is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + # embed positions + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device + ) + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length + ) + + hidden_states = inputs_embeds + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + + for idx, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + # None for past_key_value + return module(*inputs, output_attentions, None) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(decoder_layer), + hidden_states, + attention_mask, + None, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class LlamaForCausalLM(LlamaPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = LlamaModel(config) + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional + tensors are only required when the model is used as a decoder in a Sequence to Sequence model. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, LlamaForCausalLM + + >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS) + >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER) + + >>> prompt = "Hey, are you consciours? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + loss = None + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss() + loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs + ): + if past_key_values: + input_ids = input_ids[:, -1:] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past diff --git a/src/transformers/models/llama/tokenization_llama.py b/src/transformers/models/llama/tokenization_llama.py new file mode 100644 index 000000000000..521bb854f7c1 --- /dev/null +++ b/src/transformers/models/llama/tokenization_llama.py @@ -0,0 +1,231 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization classes for LLaMA.""" +import os +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple + +import sentencepiece as spm + +from ...tokenization_utils import PreTrainedTokenizer +from ...utils import logging + + +logger = logging.get_logger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"} + +PRETRAINED_VOCAB_FILES_MAP = {} + + +class LlamaTokenizer(PreTrainedTokenizer): + """ + Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + unk_token="", + bos_token="", + eos_token="", + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + decode_with_prefix_space=False, + **kwargs, + ): + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs) + self.vocab_file = vocab_file + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + self.decode_with_prefix_space = decode_with_prefix_space + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(vocab_file) + self._no_prefix_space_tokens = None + + """ Initialisation""" + + @property + def no_prefix_space_tokens(self): + if self._no_prefix_space_tokens is None: + vocab = self.convert_ids_to_tokens(list(range(self.vocab_size))) + self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith("▁")} + return self._no_prefix_space_tokens + + @property + def vocab_size(self): + """Returns vocab size""" + return self.sp_model.get_piece_size() + + @property + def bos_token_id(self) -> Optional[int]: + return self.sp_model.bos_id() + + @property + def eos_token_id(self) -> Optional[int]: + return self.sp_model.eos_id() + + def get_vocab(self): + """Returns vocab as a dict""" + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text): + """Returns a tokenized string.""" + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + token = self.sp_model.IdToPiece(index) + return token + + def _maybe_add_prefix_space(self, tokens, decoded): + if tokens and tokens[0] not in self.no_prefix_space_tokens: + return " " + decoded + else: + return decoded + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + current_sub_tokens = [] + out_string = "" + prev_is_special = False + for token in tokens: + # make sure that special tokens are not decoded using sentencepiece model + if token in self.all_special_tokens: + if not prev_is_special: + out_string += " " + out_string += self.sp_model.decode(current_sub_tokens) + token + prev_is_special = True + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + prev_is_special = False + out_string += self.sp_model.decode(current_sub_tokens) + out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string) + return out_string + + def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]: + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + out_vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, "wb") as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + if self.add_bos_token: + bos_token_ids = [self.bos_token_id] + else: + bos_token_ids = [] + + output = bos_token_ids + token_ids_0 + + if token_ids_1 is not None: + output = output + token_ids_1 + + if self.add_eos_token: + output = output + [self.eos_token_id] + + return output + + def get_special_tokens_mask( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + ) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make + use of token type ids, therefore a list of zeros is returned. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of zeros. + """ + eos = [self.eos_token_id] + + if token_ids_1 is None: + return len(token_ids_0 + eos) * [0] + return len(token_ids_0 + eos + token_ids_1 + eos) * [0] diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index fc9a8d324a61..6fb3d2d9d0bb 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -3761,6 +3761,27 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class LlamaForCausalLM(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class LlamaModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class LlamaPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/src/transformers/utils/dummy_sentencepiece_objects.py b/src/transformers/utils/dummy_sentencepiece_objects.py index 81d1e14c390b..7e0f78d6c713 100644 --- a/src/transformers/utils/dummy_sentencepiece_objects.py +++ b/src/transformers/utils/dummy_sentencepiece_objects.py @@ -86,6 +86,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["sentencepiece"]) +class LlamaTokenizer(metaclass=DummyObject): + _backends = ["sentencepiece"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["sentencepiece"]) + + class M2M100Tokenizer(metaclass=DummyObject): _backends = ["sentencepiece"] diff --git a/tests/models/llama/__init__.py b/tests/models/llama/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/llama/test_modeling_llama.py b/tests/models/llama/test_modeling_llama.py new file mode 100644 index 000000000000..23d573454da4 --- /dev/null +++ b/tests/models/llama/test_modeling_llama.py @@ -0,0 +1,304 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch LLaMA model. """ + + +import unittest + +from transformers import LlamaConfig, is_torch_available +from transformers.testing_utils import require_torch, torch_device + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask + + +if is_torch_available(): + import torch + + from transformers import LlamaForCausalLM, LlamaModel + + +class LlamaModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=False, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = random_attention_mask([self.batch_size, self.seq_length]) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = self.get_config() + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def get_config(self): + return LlamaConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + is_decoder=False, + initializer_range=self.initializer_range, + ) + + def create_and_check_model( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + model = LlamaModel(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask) + result = model(input_ids) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def create_and_check_model_as_decoder( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + config.add_cross_attention = True + model = LlamaModel(config) + model.to(torch_device) + model.eval() + result = model( + input_ids, + attention_mask=input_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + ) + result = model( + input_ids, + attention_mask=input_mask, + encoder_hidden_states=encoder_hidden_states, + ) + result = model(input_ids, attention_mask=input_mask) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def create_and_check_for_causal_lm( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + model = LlamaForCausalLM(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, labels=token_labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + + def create_and_check_decoder_model_past_large_inputs( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + config.is_decoder = True + config.add_cross_attention = True + model = LlamaForCausalLM(config=config) + model.to(torch_device) + model.eval() + + # first forward pass + outputs = model( + input_ids, + attention_mask=input_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + use_cache=True, + ) + past_key_values = outputs.past_key_values + + # create hypothetical multiple next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) + next_mask = ids_tensor((self.batch_size, 3), vocab_size=2) + + # append to next input_ids and + next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) + next_attention_mask = torch.cat([input_mask, next_mask], dim=-1) + + output_from_no_past = model( + next_input_ids, + attention_mask=next_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_hidden_states=True, + )["hidden_states"][0] + output_from_past = model( + next_tokens, + attention_mask=next_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + output_hidden_states=True, + )["hidden_states"][0] + + # select random slice + random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() + output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach() + output_from_past_slice = output_from_past[:, :, random_slice_idx].detach() + + self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]) + + # test that outputs are equal for slice + self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + ( + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + ) = config_and_inputs + inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask} + return config, inputs_dict + + +@require_torch +class LlamaModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = ( + ( + LlamaModel, + LlamaForCausalLM, + ) + if is_torch_available() + else () + ) + all_generative_model_classes = (LlamaForCausalLM,) if is_torch_available() else () + test_headmasking = False + + def setUp(self): + self.model_tester = LlamaModelTester(self) + self.config_tester = ConfigTester(self, config_class=LlamaConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_model_various_embeddings(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + for type in ["absolute", "relative_key", "relative_key_query"]: + config_and_inputs[0].position_embedding_type = type + self.model_tester.create_and_check_model(*config_and_inputs) + + @unittest.skip("LLaMA does not support head pruning.") + def test_head_pruning(self): + pass + + @unittest.skip("LLaMA does not support head pruning.") + def test_head_pruning_integration(self): + pass + + @unittest.skip("LLaMA does not support head pruning.") + def test_head_pruning_save_load_from_config_init(self): + pass + + @unittest.skip("LLaMA does not support head pruning.") + def test_head_pruning_save_load_from_pretrained(self): + pass + + @unittest.skip("LLaMA buffers include complex numbers, which breaks this test") + def test_save_load_fast_init_from_base(self): + pass diff --git a/utils/check_config_docstrings.py b/utils/check_config_docstrings.py index 94066ff76455..90585095fc09 100644 --- a/utils/check_config_docstrings.py +++ b/utils/check_config_docstrings.py @@ -41,6 +41,7 @@ "SpeechEncoderDecoderConfig", "VisionEncoderDecoderConfig", "VisionTextDualEncoderConfig", + "LlamaConfig", } diff --git a/utils/check_repo.py b/utils/check_repo.py index c59897d832b5..121993bc1e83 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -57,6 +57,7 @@ # Being in this list is an exception and should **not** be the rule. IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [ # models to ignore for not tested + "LlamaDecoder", # Building part of bigger (tested) model. "Blip2QFormerModel", # Building part of bigger (tested) model. "DetaEncoder", # Building part of bigger (tested) model. "DetaDecoder", # Building part of bigger (tested) model. From 45d77418189a91d10c64f922bb958b6d5b98ba20 Mon Sep 17 00:00:00 2001 From: Jason Phang Date: Thu, 16 Mar 2023 09:01:15 -0400 Subject: [PATCH 112/277] LLaMA Implementation (#21955) * LLaMA * sharding and docs * tweak * black * inits * ruff * LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP * init * no checkpoint * docs * ruff * type_vocab_size * tokenizer fixes * tokenizer fixes * Update tokenization_llama.py * Update tokenization_llama.py * Update configuration_llama.py * Update modeling_llama.py * tokenizer add_bos by default * licenses * remove decoder * norms and mlp * rope overhaul * tweaks * black * mention OPT implementation * off-by-one naming * typo * fix * tokenization fix and slicing bug * padding config * cleanup * black * update tests * undo typo * fix vocab caching logic * ruff * docbuilder * attn fix from BlackSamorez * initial feedback * typo * docs * llama case * llama case * load checkpoint docs * comment about tokenizer * tokenizer defaults * clear past_key_values if use_cache=False * last tweaks * last tweaks * last tweaks * last tweaks --------- Co-authored-by: Stella Biderman From fdce70e822613e8defebe9c59ea4bcbdcdaeaca3 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 16 Mar 2023 14:21:58 +0100 Subject: [PATCH 113/277] Update tiny model creation script (#22202) * Update UNCONVERTIBLE_MODEL_ARCHITECTURES * Deal with 2 model tester classes in single test file * Deal with 2 model tester classes in single test file * Deal with 2 model tester classes in single test file * make style and quality --------- Co-authored-by: ydshieh --- utils/create_dummy_models.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/utils/create_dummy_models.py b/utils/create_dummy_models.py index 176bd07dfaaa..975c0703d4c4 100644 --- a/utils/create_dummy_models.py +++ b/utils/create_dummy_models.py @@ -16,18 +16,17 @@ import argparse import collections.abc import copy -import importlib import inspect import json import os import shutil -import sys import tempfile import traceback from pathlib import Path from check_config_docstrings import get_checkpoint_from_config_class from datasets import load_dataset +from get_test_info import get_model_to_tester_mapping, get_tester_classes_for_model from huggingface_hub import Repository, create_repo, upload_folder from transformers import ( @@ -58,7 +57,6 @@ logging.disable_progress_bar() logger = logging.get_logger(__name__) -sys.path.append(".") os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" if not is_torch_available(): @@ -67,6 +65,7 @@ if not is_tf_available(): raise ValueError("Please install TensorFlow.") + FRAMEWORKS = ["pytorch", "tensorflow"] INVALID_ARCH = [] TARGET_VOCAB_SIZE = 1024 @@ -94,8 +93,12 @@ "TFCamembertModel", "TFCamembertForCausalLM", "DecisionTransformerModel", + "GraphormerModel", + "InformerModel", "JukeboxModel", "MarianForCausalLM", + "MaskFormerSwinModel", + "MaskFormerSwinBackbone", "MT5Model", "MT5ForConditionalGeneration", "TFMT5ForConditionalGeneration", @@ -126,6 +129,7 @@ "XLMRobertaForQuestionAnswering", "TFXLMRobertaForSequenceClassification", "TFXLMRobertaForMaskedLM", + "TFXLMRobertaForCausalLM", "TFXLMRobertaForQuestionAnswering", "TFXLMRobertaModel", "TFXLMRobertaForMultipleChoice", @@ -355,7 +359,7 @@ def build_processor(config_class, processor_class, allow_no_checkpoint=False): return processor -def get_tiny_config(config_class, **model_tester_kwargs): +def get_tiny_config(config_class, model_class=None, **model_tester_kwargs): """Retrieve a tiny configuration from `config_class` using each model's `ModelTester`. Args: @@ -378,9 +382,18 @@ def get_tiny_config(config_class, **model_tester_kwargs): module_name = model_type_to_module_name(model_type) if not modeling_name.startswith(module_name): raise ValueError(f"{modeling_name} doesn't start with {module_name}!") - module = importlib.import_module(f".models.{module_name}.test_modeling_{modeling_name}", package="tests") - camel_case_model_name = config_class.__name__.split("Config")[0] - model_tester_class = getattr(module, f"{camel_case_model_name}ModelTester", None) + test_file = os.path.join("tests", "models", module_name, f"test_modeling_{modeling_name}.py") + models_to_model_testers = get_model_to_tester_mapping(test_file) + # Find the model tester class + model_tester_class = None + tester_classes = [] + if model_class is not None: + tester_classes = get_tester_classes_for_model(test_file, model_class) + else: + for _tester_classes in models_to_model_testers.values(): + tester_classes.extend(_tester_classes) + if len(tester_classes) > 0: + model_tester_class = sorted(tester_classes, key=lambda x: x.__name__)[0] except ModuleNotFoundError: error = f"Tiny config not created for {model_type} - cannot find the testing module from the model name." raise ValueError(error) From 0bb5eb9787b3033e3ea1720b62f3065c3cf12755 Mon Sep 17 00:00:00 2001 From: SatyaJandhyalaAtMS <55203776+SatyaJandhyalaAtMS@users.noreply.github.com> Date: Thu, 16 Mar 2023 07:56:26 -0700 Subject: [PATCH 114/277] Temporarily fix ONNX model exporting error (#21830) * Temporarily fix https://github.com/microsoft/onnx-converters-private/issues/143 * Reduced column width * Fix formatting. * Revert "Temporarily fix https://github.com/microsoft/onnx-converters-private/issues/143" This reverts commit 6e95a108042118d204da447729f3834affa354fc. * Fix export error. * Revert "Fix formatting." This reverts commit 8310f60da10358edbdf77a2a2f3c83ee55066cb8. * Propagated changes made in SwinV2 to Swin2SR --- src/transformers/models/swin2sr/modeling_swin2sr.py | 5 ++--- src/transformers/models/swinv2/modeling_swinv2.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/swin2sr/modeling_swin2sr.py b/src/transformers/models/swin2sr/modeling_swin2sr.py index 256b8decda7e..cd58b7065058 100644 --- a/src/transformers/models/swin2sr/modeling_swin2sr.py +++ b/src/transformers/models/swin2sr/modeling_swin2sr.py @@ -520,9 +520,8 @@ def set_shift_and_window_size(self, input_resolution): if isinstance(self.shift_size, collections.abc.Iterable) else (self.shift_size, self.shift_size) ) - self.window_size = ( - input_resolution[0] if input_resolution[0] <= target_window_size[0] else target_window_size[0] - ) + window_dim = input_resolution[0].item() if torch.is_tensor(input_resolution[0]) else input_resolution[0] + self.window_size = window_dim if window_dim <= target_window_size[0] else target_window_size[0] self.shift_size = ( 0 if input_resolution diff --git a/src/transformers/models/swinv2/modeling_swinv2.py b/src/transformers/models/swinv2/modeling_swinv2.py index 3104e5d2d20e..21d61c4e5d65 100644 --- a/src/transformers/models/swinv2/modeling_swinv2.py +++ b/src/transformers/models/swinv2/modeling_swinv2.py @@ -662,9 +662,8 @@ def set_shift_and_window_size(self, input_resolution): if isinstance(self.shift_size, collections.abc.Iterable) else (self.shift_size, self.shift_size) ) - self.window_size = ( - input_resolution[0] if input_resolution[0] <= target_window_size[0] else target_window_size[0] - ) + window_dim = input_resolution[0].item() if torch.is_tensor(input_resolution[0]) else input_resolution[0] + self.window_size = window_dim if window_dim <= target_window_size[0] else target_window_size[0] self.shift_size = ( 0 if input_resolution From a0e90aed54ef2ebec833f48a86add5a41c904475 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:18:05 +0100 Subject: [PATCH 115/277] [`XGLM`] Add `accelerate` support for XGLM (#22207) * add `accelerate` support for XGLM * fix order --- src/transformers/models/xglm/modeling_xglm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/xglm/modeling_xglm.py b/src/transformers/models/xglm/modeling_xglm.py index b9cef18efc7e..8a19557932f9 100755 --- a/src/transformers/models/xglm/modeling_xglm.py +++ b/src/transformers/models/xglm/modeling_xglm.py @@ -510,6 +510,7 @@ class XGLMPreTrainedModel(PreTrainedModel): config_class = XGLMConfig base_model_prefix = "model" supports_gradient_checkpointing = True + _no_split_modules = ["XGLMDecoderLayer"] def _init_weights(self, module): std = self.config.init_std From ff4142ab0877e71538872b27cc0898ab167ed30b Mon Sep 17 00:00:00 2001 From: Susnato Dhar Date: Thu, 16 Mar 2023 21:38:05 +0530 Subject: [PATCH 116/277] fixes a typo in WhisperFeatureExtractor docs. (#22208) * fixes a typo * . --- src/transformers/models/whisper/feature_extraction_whisper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/whisper/feature_extraction_whisper.py b/src/transformers/models/whisper/feature_extraction_whisper.py index 3ea719d0da79..da700f2d2257 100644 --- a/src/transformers/models/whisper/feature_extraction_whisper.py +++ b/src/transformers/models/whisper/feature_extraction_whisper.py @@ -33,8 +33,8 @@ class WhisperFeatureExtractor(SequenceFeatureExtractor): r""" Constructs a Whisper feature extractor. - This feature extractor inherits from [`WhisperFeatureExtractor`] which contains most of the main methods. Users - should refer to this superclass for more information regarding those methods. + This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains + most of the main methods. Users should refer to this superclass for more information regarding those methods. This class extracts mel-filter bank features from raw speech using a custom numpy implementation of the `Short Time Fourier Transform` which should match pytorch's `torch.stft` equivalent. From b24b1a5cab5892d2f20f43c72a3ec08bf20adb6e Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 16 Mar 2023 22:59:23 +0100 Subject: [PATCH 117/277] =?UTF-8?q?=F0=9F=94=A5py38=20+=20torch=202=20?= =?UTF-8?q?=F0=9F=94=A5=F0=9F=94=A5=F0=9F=94=A5=F0=9F=9A=80=20(#22204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * py38 + torch 2 * increment cache versions --------- Co-authored-by: ydshieh --- .circleci/config.yml | 18 +++++++++--------- .circleci/create_circleci_config.py | 4 ++-- tests/generation/test_utils.py | 2 +- tests/onnx/test_onnx_v2.py | 2 ++ ...pipelines_zero_shot_image_classification.py | 9 +++++++-- tests/trainer/test_trainer.py | 1 + 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 736f61736df6..7dd66a97d72a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: # Ensure running with CircleCI/huggingface check_circleci_user: docker: - - image: cimg/python:3.7.12 + - image: cimg/python:3.8.12 parallelism: 1 steps: - run: echo $CIRCLE_PROJECT_USERNAME @@ -26,7 +26,7 @@ jobs: fetch_tests: working_directory: ~/transformers docker: - - image: cimg/python:3.7.12 + - image: cimg/python:3.8.12 parallelism: 1 steps: - checkout @@ -85,7 +85,7 @@ jobs: fetch_all_tests: working_directory: ~/transformers docker: - - image: cimg/python:3.7.12 + - image: cimg/python:3.8.12 parallelism: 1 steps: - checkout @@ -111,7 +111,7 @@ jobs: check_code_quality: working_directory: ~/transformers docker: - - image: cimg/python:3.7.12 + - image: cimg/python:3.8.12 resource_class: large environment: TRANSFORMERS_IS_CI: yes @@ -121,8 +121,8 @@ jobs: - checkout - restore_cache: keys: - - v0.5-code_quality-{{ checksum "setup.py" }} - - v0.5-code-quality + - v0.6-code_quality-{{ checksum "setup.py" }} + - v0.6-code-quality - run: pip install --upgrade pip - run: pip install .[all,quality] - save_cache: @@ -144,7 +144,7 @@ jobs: check_repository_consistency: working_directory: ~/transformers docker: - - image: cimg/python:3.7.12 + - image: cimg/python:3.8.12 resource_class: large environment: TRANSFORMERS_IS_CI: yes @@ -154,8 +154,8 @@ jobs: - checkout - restore_cache: keys: - - v0.5-repository_consistency-{{ checksum "setup.py" }} - - v0.5-repository_consistency + - v0.6-repository_consistency-{{ checksum "setup.py" }} + - v0.6-repository_consistency - run: pip install --upgrade pip - run: pip install .[all,quality] - save_cache: diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index d67968d5afec..0b26762b080a 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -33,7 +33,7 @@ "RUN_PT_FLAX_CROSS_TESTS": False, } COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile", "s": None} -DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.7.12"}] +DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.8.12"}] @dataclass @@ -41,7 +41,7 @@ class CircleCIJob: name: str additional_env: Dict[str, Any] = None cache_name: str = None - cache_version: str = "0.5" + cache_version: str = "0.6" docker_image: List[Dict[str, str]] = None install_steps: List[str] = None marker: Optional[str] = None diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index b0d23b6fff2f..de319c2b0004 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -2450,7 +2450,7 @@ def test_eos_token_id_int_and_list_top_k_top_sampling(self): "top_k": 10, "temperature": 0.7, } - expectation = 15 + expectation = 20 tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") text = """Hello, my dog is cute and""" diff --git a/tests/onnx/test_onnx_v2.py b/tests/onnx/test_onnx_v2.py index 792c2a65cb0c..51d9e2cb1843 100644 --- a/tests/onnx/test_onnx_v2.py +++ b/tests/onnx/test_onnx_v2.py @@ -1,4 +1,5 @@ import os +import unittest from pathlib import Path from tempfile import NamedTemporaryFile from unittest import TestCase @@ -499,6 +500,7 @@ def test_tensorflow_export_seq2seq_with_past( class StableDropoutTestCase(TestCase): """Tests export of StableDropout module.""" + @unittest.skip("torch 2.0.0 gives `torch.onnx.errors.OnnxExporterError: Module onnx is not installed!`.") @require_torch @pytest.mark.filterwarnings("ignore:.*Dropout.*:UserWarning:torch.onnx.*") # torch.onnx is spammy. def test_training(self): diff --git a/tests/pipelines/test_pipelines_zero_shot_image_classification.py b/tests/pipelines/test_pipelines_zero_shot_image_classification.py index fb101f421042..fbbfc78cae3c 100644 --- a/tests/pipelines/test_pipelines_zero_shot_image_classification.py +++ b/tests/pipelines/test_pipelines_zero_shot_image_classification.py @@ -78,9 +78,14 @@ def test_small_model_pt(self): image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") output = image_classifier(image, candidate_labels=["a", "b", "c"]) - self.assertEqual( + # The floating scores are so close, we enter floating error approximation and the order is not guaranteed across + # python and torch versions. + self.assertIn( nested_simplify(output), - [{"score": 0.333, "label": "a"}, {"score": 0.333, "label": "b"}, {"score": 0.333, "label": "c"}], + [ + [{"score": 0.333, "label": "a"}, {"score": 0.333, "label": "b"}, {"score": 0.333, "label": "c"}], + [{"score": 0.333, "label": "a"}, {"score": 0.333, "label": "c"}, {"score": 0.333, "label": "b"}], + ], ) output = image_classifier([image] * 5, candidate_labels=["A", "B", "C"], batch_size=2) diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 274ee76e53c8..310842713bde 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -1855,6 +1855,7 @@ def test_torchdynamo_full_eval(self): self.assertAlmostEqual(metrics["eval_loss"], original_eval_loss) torchdynamo.reset() + @unittest.skip("torch 2.0.0 gives `ModuleNotFoundError: No module named 'torchdynamo'`.") @require_torch_non_multi_gpu @require_torchdynamo def test_torchdynamo_memory(self): From d10e67fb57e6873c1d571394e4a0d9c92c9ab8a9 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 16 Mar 2023 23:57:26 +0100 Subject: [PATCH 118/277] Hotfix for natten issue with torch 2.0.0 on CircleCI (#22218) fix Co-authored-by: ydshieh --- .circleci/create_circleci_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 0b26762b080a..6a45f94a6578 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -374,7 +374,8 @@ def job_name(self): "pip install 'git+https://github.com/facebookresearch/detectron2.git'", "sudo apt install tesseract-ocr", "pip install pytesseract", - "pip install natten", + # wait until natten is ready for torch 2.0.0 + # "pip install natten", ], tests_to_run=[ "tests/models/*layoutlmv*", From bbb5691371023a3b0f43a80d97b9bf8a34bb00c0 Mon Sep 17 00:00:00 2001 From: Kevin Turner <83819+keturn@users.noreply.github.com> Date: Fri, 17 Mar 2023 01:43:18 -0700 Subject: [PATCH 119/277] fix typos in llama.mdx (#22223) --- docs/source/en/model_doc/llama.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/llama.mdx b/docs/source/en/model_doc/llama.mdx index 3f6ea3409bb8..28e39b11ce60 100644 --- a/docs/source/en/model_doc/llama.mdx +++ b/docs/source/en/model_doc/llama.mdx @@ -22,7 +22,7 @@ The abstract from the paper is the following: Tips: -- Weights for the LLaMA models can be obtained from by filling out [this form]()https://docs.google.com/forms/d/e/1FAIpQLSfqNECQnMkycAp2jP4Z9TFX0cGR4uf7b_fBxjY_OjhJILlKGA/viewform?usp=send_form) +- Weights for the LLaMA models can be obtained from by filling out [this form](https://docs.google.com/forms/d/e/1FAIpQLSfqNECQnMkycAp2jP4Z9TFX0cGR4uf7b_fBxjY_OjhJILlKGA/viewform?usp=send_form) - After downloading the weights, they will need to be converted to the Hugging Face Transformers format using the [conversion script](/src/transformers/models/llama/convert_llama_weights_to_hf.py). The script can be called with the following (example) command: ```bash @@ -37,7 +37,7 @@ tokenizer = transformers.LlamaTokenizer.from_pretrained("/output/path/tokenizer/ model = transformers.LlamaForCausalLM.from_pretrained("/output/path/llama-7b/") ``` -- The LLaMA tokenizer is based on [sentencepiece](https://github.com/google/sentencepiece). One quick of sentencepiece is that when decoding a sequence, if the first token is the start of the word (e.g. "Banana"), the tokenizer does not prepend the prefix space to the string. To have the tokenizer output the prefix space, set `decode_with_prefix_space=True` in the `LlamaTokenizer` object or in the tokenizer configuration. +- The LLaMA tokenizer is based on [sentencepiece](https://github.com/google/sentencepiece). One quirk of sentencepiece is that when decoding a sequence, if the first token is the start of the word (e.g. "Banana"), the tokenizer does not prepend the prefix space to the string. To have the tokenizer output the prefix space, set `decode_with_prefix_space=True` in the `LlamaTokenizer` object or in the tokenizer configuration. This model was contributed by [zphang](https://huggingface.co/zphang) with contributions from [BlackSamorez](https://huggingface.co/BlackSamorez). The code of the implementation in Hugging Face is based on GPT-NeoX [here](https://github.com/EleutherAI/gpt-neox). The original code of the authors can be found [here](https://github.com/facebookresearch/llama). From 0a879304ce81b99ac92ecc556bf769a83fb7070c Mon Sep 17 00:00:00 2001 From: wangpeng <1204591829@qq.com> Date: Fri, 17 Mar 2023 17:40:06 +0800 Subject: [PATCH 120/277] fix code example in mgp-str doc (#22219) Co-authored-by: yue kun --- docs/source/en/model_doc/mgp-str.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/model_doc/mgp-str.mdx b/docs/source/en/model_doc/mgp-str.mdx index fc3d9f41526d..77c937bfb2b6 100644 --- a/docs/source/en/model_doc/mgp-str.mdx +++ b/docs/source/en/model_doc/mgp-str.mdx @@ -54,7 +54,7 @@ into a single instance to both extract the input features and decode the predict >>> url = "https://i.postimg.cc/ZKwLg2Gw/367-14.png" >>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB") ->>> pixel_values = processor(image, return_tensors="pt").pixel_values +>>> pixel_values = processor(images=image, return_tensors="pt").pixel_values >>> outputs = model(pixel_values) >>> generated_text = processor.batch_decode(outputs.logits)['generated_text'] From e0571080a124c89fb9086b131ad9867490ac43ed Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 17 Mar 2023 13:27:14 +0100 Subject: [PATCH 121/277] Use `dash==2.8.1` for now for daily CI (#22227) Use dash 2.8.1 for now Co-authored-by: ydshieh --- docker/transformers-all-latest-gpu/Dockerfile | 3 +++ docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile | 3 +++ docker/transformers-pytorch-gpu/Dockerfile | 3 +++ 3 files changed, 9 insertions(+) diff --git a/docker/transformers-all-latest-gpu/Dockerfile b/docker/transformers-all-latest-gpu/Dockerfile index 9f30f4531b0a..da2ed78bd080 100644 --- a/docker/transformers-all-latest-gpu/Dockerfile +++ b/docker/transformers-all-latest-gpu/Dockerfile @@ -59,6 +59,9 @@ RUN python3 -m pip install --no-cache-dir decord av==9.2.0 ## For `dinat` model #RUN python3 -m pip install --no-cache-dir natten -f https://shi-labs.com/natten/wheels/$CUDA/ +# dash 2.9.0 has some issue. +RUN python3 -m pip install -U dash==2.8.1 + # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile index 2fa1317f8275..09160b96aad6 100644 --- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile +++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile @@ -40,6 +40,9 @@ RUN python3 -m pip uninstall -y deepspeed # TODO: Find out why test fail. RUN DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1 +# dash 2.9.0 has some issue. +RUN python3 -m pip install -U dash==2.8.1 + # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop diff --git a/docker/transformers-pytorch-gpu/Dockerfile b/docker/transformers-pytorch-gpu/Dockerfile index 24564aca6fae..a5115890a763 100644 --- a/docker/transformers-pytorch-gpu/Dockerfile +++ b/docker/transformers-pytorch-gpu/Dockerfile @@ -29,6 +29,9 @@ RUN python3 -m pip uninstall -y tensorflow flax RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract RUN python3 -m pip install -U "itsdangerous<2.1.0" +# dash 2.9.0 has some issue. +RUN python3 -m pip install -U dash==2.8.1 + # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop From 98613dc96fc2138d5c0c01d6e87e5aadcb294830 Mon Sep 17 00:00:00 2001 From: Maria Khalusova Date: Fri, 17 Mar 2023 08:36:23 -0400 Subject: [PATCH 122/277] Depth estimation task guide (#22205) * added doc to toc, auto tip with supported models, mention of task guide in model docs * make style * removed "see also" * minor fix --- docs/source/en/_toctree.yml | 2 + docs/source/en/model_doc/dpt.mdx | 3 +- docs/source/en/model_doc/glpn.mdx | 1 + .../en/tasks/monocular_depth_estimation.mdx | 147 ++++++++++++++++++ utils/check_task_guides.py | 1 + 5 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 docs/source/en/tasks/monocular_depth_estimation.mdx diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index d8833f3efe08..7586985b1112 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -85,6 +85,8 @@ title: Zero-shot object detection - local: tasks/zero_shot_image_classification title: Zero-shot image classification + - local: tasks/monocular_depth_estimation + title: Depth estimation title: Computer Vision - sections: - local: tasks/image_captioning diff --git a/docs/source/en/model_doc/dpt.mdx b/docs/source/en/model_doc/dpt.mdx index b19a7468e633..c80ab8db7c0e 100644 --- a/docs/source/en/model_doc/dpt.mdx +++ b/docs/source/en/model_doc/dpt.mdx @@ -33,7 +33,8 @@ This model was contributed by [nielsr](https://huggingface.co/nielsr). The origi A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with DPT. - Demo notebooks for [`DPTForDepthEstimation`] can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/DPT). -- See also: [Semantic segmentation task guide](./tasks/semantic_segmentation) +- [Semantic segmentation task guide](../tasks/semantic_segmentation) +- [Monocular depth estimation task guide](../tasks/monocular_depth_estimation.mdx) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/glpn.mdx b/docs/source/en/model_doc/glpn.mdx index fe39dbb9489a..ce930b2e95a2 100644 --- a/docs/source/en/model_doc/glpn.mdx +++ b/docs/source/en/model_doc/glpn.mdx @@ -45,6 +45,7 @@ This model was contributed by [nielsr](https://huggingface.co/nielsr). The origi A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with GLPN. - Demo notebooks for [`GLPNForDepthEstimation`] can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/GLPN). +- [Monocular depth estimation task guide](../tasks/monocular_depth_estimation.mdx) ## GLPNConfig diff --git a/docs/source/en/tasks/monocular_depth_estimation.mdx b/docs/source/en/tasks/monocular_depth_estimation.mdx new file mode 100644 index 000000000000..a2721d659e6e --- /dev/null +++ b/docs/source/en/tasks/monocular_depth_estimation.mdx @@ -0,0 +1,147 @@ + + +# Monocular depth estimation + +Monocular depth estimation is a computer vision task that involves predicting the depth information of a scene from a +single image. In other words, it is the process of estimating the distance of objects in a scene from +a single camera viewpoint. + +Monocular depth estimation has various applications, including 3D reconstruction, augmented reality, autonomous driving, +and robotics. It is a challenging task as it requires the model to understand the complex relationships between objects +in the scene and the corresponding depth information, which can be affected by factors such as lighting conditions, +occlusion, and texture. + + +The task illustrated in this tutorial is supported by the following model architectures: + + + +[DPT](../model_doc/dpt), [GLPN](../model_doc/glpn) + + + + + +In this guide you'll learn how to: + +* create a depth estimation pipeline +* run depth estimation inference by hand + +Before you begin, make sure you have all the necessary libraries installed: + +```bash +pip install -q transformers +``` + +## Depth estimation pipeline + +The simplest way to try out inference with a model supporting depth estimation is to use the corresponding [`pipeline`]. +Instantiate a pipeline from a [checkpoint on the Hugging Face Hub](https://huggingface.co/models?pipeline_tag=depth-estimation&sort=downloads): + +```py +>>> from transformers import pipeline + +>>> checkpoint = "vinvino02/glpn-nyu" +>>> depth_estimator = pipeline("depth-estimation", model=checkpoint) +``` + +Next, choose an image to analyze: + +```py +>>> from PIL import Image +>>> import requests + +>>> url = "https://unsplash.com/photos/HwBAsSbPBDU/download?ixid=MnwxMjA3fDB8MXxzZWFyY2h8MzR8fGNhciUyMGluJTIwdGhlJTIwc3RyZWV0fGVufDB8MHx8fDE2Nzg5MDEwODg&force=true&w=640" +>>> image = Image.open(requests.get(url, stream=True).raw) +>>> image +``` + +
+ Photo of a busy street +
+ +Pass the image to the pipeline. + +```py +>>> predictions = depth_estimator(image) +``` + +The pipeline returns a dictionary with two entries. The first one, called `predicted_depth`, is a tensor with the values +being the depth expressed in meters for each pixel. +The second one, `depth`, is a PIL image that visualizes the depth estimation result. + +Let's take a look at the visualized result: + +```py +>>> predictions["depth"] +``` + +
+ Depth estimation visualization +
+ +## Depth estimation inference by hand + +Now that you've seen how to use the depth estimation pipeline, let's see how we can replicate the same result by hand. + +Start by loading the model and associated processor from a [checkpoint on the Hugging Face Hub](https://huggingface.co/models?pipeline_tag=depth-estimation&sort=downloads). +Here we'll use the same checkpoint as before: + +```py +>>> from transformers import AutoImageProcessor, AutoModelForDepthEstimation + +>>> checkpoint = "vinvino02/glpn-nyu" + +>>> image_processor = AutoImageProcessor.from_pretrained(checkpoint) +>>> model = AutoModelForDepthEstimation.from_pretrained(checkpoint) +``` + +Prepare the image input for the model using the `image_processor` that will take care of the necessary image transformations +such as resizing and normalization: + +```py +>>> pixel_values = image_processor(image, return_tensors="pt").pixel_values +``` + +Pass the prepared inputs through the model: + +```py +>>> import torch + +>>> with torch.no_grad(): +... outputs = model(pixel_values) +... predicted_depth = outputs.predicted_depth +``` + +Visualize the results: + +```py +>>> import numpy as np + +>>> # interpolate to original size +>>> prediction = torch.nn.functional.interpolate( +... predicted_depth.unsqueeze(1), +... size=image.size[::-1], +... mode="bicubic", +... align_corners=False, +... ).squeeze() +>>> output = prediction.numpy() + +>>> formatted = (output * 255 / np.max(output)).astype("uint8") +>>> depth = Image.fromarray(formatted) +>>> depth +``` + +
+ Depth estimation visualization +
diff --git a/utils/check_task_guides.py b/utils/check_task_guides.py index 380012341685..42515439a98f 100644 --- a/utils/check_task_guides.py +++ b/utils/check_task_guides.py @@ -70,6 +70,7 @@ def _find_text_in_file(filename, start_prompt, end_prompt): "translation.mdx": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES, "video_classification.mdx": transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES, "document_question_answering.mdx": transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES, + "monocular_depth_estimation.mdx": transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES, } # This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any From a6a030d770b829781faa7a15ab8db83b1ec79aab Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Fri, 17 Mar 2023 08:55:15 -0400 Subject: [PATCH 123/277] LLaMA house-keeping (#22216) * LLaMA house-keeping * Doc links --- docs/source/en/model_doc/llama.mdx | 6 ++++-- src/transformers/__init__.py | 2 +- src/transformers/models/llama/configuration_llama.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/source/en/model_doc/llama.mdx b/docs/source/en/model_doc/llama.mdx index 28e39b11ce60..27eee085e6ad 100644 --- a/docs/source/en/model_doc/llama.mdx +++ b/docs/source/en/model_doc/llama.mdx @@ -33,8 +33,10 @@ python src/transformers/models/llama/convert_llama_weights_to_hf.py \ - After conversion, the model and tokenizer can be loaded via: ```python -tokenizer = transformers.LlamaTokenizer.from_pretrained("/output/path/tokenizer/") -model = transformers.LlamaForCausalLM.from_pretrained("/output/path/llama-7b/") +from transformers import LlamaForCausalLM, LlamaTokenizer + +tokenizer = LlamaTokenizer.from_pretrained("/output/path/tokenizer/") +model = LlamaForCausalLM.from_pretrained("/output/path/llama-7b/") ``` - The LLaMA tokenizer is based on [sentencepiece](https://github.com/google/sentencepiece). One quirk of sentencepiece is that when decoding a sequence, if the first token is the start of the word (e.g. "Banana"), the tokenizer does not prepend the prefix space to the string. To have the tokenizer output the prefix space, set `decode_with_prefix_space=True` in the `LlamaTokenizer` object or in the tokenizer configuration. diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 1bc0747ac7f1..cfa2de4cbf40 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -4487,9 +4487,9 @@ TypicalLogitsWarper, top_k_top_p_filtering, ) + from .modeling_utils import PreTrainedModel # PyTorch model imports - from .modeling_utils import PreTrainedModel from .models.albert import ( ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST, AlbertForMaskedLM, diff --git a/src/transformers/models/llama/configuration_llama.py b/src/transformers/models/llama/configuration_llama.py index 5421d429b101..36b8ab72ab2c 100644 --- a/src/transformers/models/llama/configuration_llama.py +++ b/src/transformers/models/llama/configuration_llama.py @@ -30,7 +30,7 @@ class LlamaConfig(PretrainedConfig): r""" - This is the configuration class to store the configuration of a [`~LlamaModel`]. It is used to instantiate an LLaMA + This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the LLaMA-7B. @@ -41,7 +41,7 @@ class LlamaConfig(PretrainedConfig): Args: vocab_size (`int`, *optional*, defaults to 32000): Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`~LlamaModel`] + `inputs_ids` passed when calling [`LlamaModel`] hidden_size (`int`, *optional*, defaults to 4096): Dimension of the hidden representations. intermediate_size (`int`, *optional*, defaults to 11008): From 2c73d7babf0a1b6cb9ac0614f842202fff6b0226 Mon Sep 17 00:00:00 2001 From: "Wang, Yi" Date: Fri, 17 Mar 2023 21:28:17 +0800 Subject: [PATCH 124/277] fix AutoTP in deepspeed could not work for bloom (#22196) * fix AutoTP in deepspeed could not work for bloom Signed-off-by: Wang, Yi A * add a method in BloomModel to build ailib Signed-off-by: Wang, Yi A --------- Signed-off-by: Wang, Yi A --- src/transformers/models/bloom/modeling_bloom.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/bloom/modeling_bloom.py b/src/transformers/models/bloom/modeling_bloom.py index 6cc0ef7c9947..f598c8299d15 100644 --- a/src/transformers/models/bloom/modeling_bloom.py +++ b/src/transformers/models/bloom/modeling_bloom.py @@ -641,6 +641,9 @@ def __init__(self, config: BloomConfig): # Initialize weights and apply final processing self.post_init() + def build_alibi_tensor(self, attention_mask: torch.Tensor, num_heads: int, dtype: torch.dtype) -> torch.Tensor: + return build_alibi_tensor(attention_mask, num_heads, dtype) + def get_input_embeddings(self): return self.word_embeddings @@ -750,7 +753,7 @@ def forward( else: attention_mask = attention_mask.to(hidden_states.device) - alibi = build_alibi_tensor(attention_mask, self.num_heads, dtype=hidden_states.dtype) + alibi = self.build_alibi_tensor(attention_mask, self.num_heads, dtype=hidden_states.dtype) causal_mask = self._prepare_attn_mask( attention_mask, From a09ef84f5c0f57cc5882ec232c197017c5b201c6 Mon Sep 17 00:00:00 2001 From: lewtun Date: Fri, 17 Mar 2023 14:39:26 +0100 Subject: [PATCH 125/277] Add LlamaForSequenceClassification (#22209) * Add LlamaForSequenceClassification * Update src/transformers/models/llama/modeling_llama.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> * Update src/transformers/models/llama/modeling_llama.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> * Add docstring * Add test * Add input embedding getter and setter * Remove dead code --------- Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- docs/source/en/model_doc/llama.mdx | 5 + .../en/tasks/sequence_classification.mdx | 2 +- src/transformers/__init__.py | 12 +- src/transformers/models/auto/modeling_auto.py | 1 + src/transformers/models/llama/__init__.py | 7 +- .../models/llama/modeling_llama.py | 134 ++++++++++++++++-- src/transformers/utils/dummy_pt_objects.py | 7 + tests/models/llama/test_modeling_llama.py | 51 +++++-- 8 files changed, 183 insertions(+), 36 deletions(-) diff --git a/docs/source/en/model_doc/llama.mdx b/docs/source/en/model_doc/llama.mdx index 27eee085e6ad..92c03be29f62 100644 --- a/docs/source/en/model_doc/llama.mdx +++ b/docs/source/en/model_doc/llama.mdx @@ -66,3 +66,8 @@ This model was contributed by [zphang](https://huggingface.co/zphang) with contr [[autodoc]] LlamaForCausalLM - forward + +## LlamaForSequenceClassification + +[[autodoc]] LlamaForSequenceClassification + - forward \ No newline at end of file diff --git a/docs/source/en/tasks/sequence_classification.mdx b/docs/source/en/tasks/sequence_classification.mdx index fb80047dd1e9..20a3f4d14686 100644 --- a/docs/source/en/tasks/sequence_classification.mdx +++ b/docs/source/en/tasks/sequence_classification.mdx @@ -28,7 +28,7 @@ The task illustrated in this tutorial is supported by the following model archit -[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT-J](../model_doc/gptj), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LED](../model_doc/led), [LiLT](../model_doc/lilt), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Perceiver](../model_doc/perceiver), [PLBart](../model_doc/plbart), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [TAPAS](../model_doc/tapas), [Transformer-XL](../model_doc/transfo-xl), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) +[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT-J](../model_doc/gptj), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LED](../model_doc/led), [LiLT](../model_doc/lilt), [LLaMA](../model_doc/llama), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Perceiver](../model_doc/perceiver), [PLBart](../model_doc/plbart), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [TAPAS](../model_doc/tapas), [Transformer-XL](../model_doc/transfo-xl), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index cfa2de4cbf40..7b70b4ac1f3b 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1801,11 +1801,7 @@ ] ) _import_structure["models.llama"].extend( - [ - "LlamaForCausalLM", - "LlamaModel", - "LlamaPreTrainedModel", - ] + ["LlamaForCausalLM", "LlamaForSequenceClassification", "LlamaModel", "LlamaPreTrainedModel"] ) _import_structure["models.longformer"].extend( [ @@ -5199,11 +5195,7 @@ LiltModel, LiltPreTrainedModel, ) - from .models.llama import ( - LlamaForCausalLM, - LlamaModel, - LlamaPreTrainedModel, - ) + from .models.llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel from .models.longformer import ( LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST, LongformerForMaskedLM, diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 6da804733a5f..eff11b45a53f 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -652,6 +652,7 @@ ("layoutlmv3", "LayoutLMv3ForSequenceClassification"), ("led", "LEDForSequenceClassification"), ("lilt", "LiltForSequenceClassification"), + ("llama", "LlamaForSequenceClassification"), ("longformer", "LongformerForSequenceClassification"), ("luke", "LukeForSequenceClassification"), ("markuplm", "MarkupLMForSequenceClassification"), diff --git a/src/transformers/models/llama/__init__.py b/src/transformers/models/llama/__init__.py index 2501c282f0ba..adef4306f287 100644 --- a/src/transformers/models/llama/__init__.py +++ b/src/transformers/models/llama/__init__.py @@ -43,6 +43,7 @@ "LlamaForCausalLM", "LlamaModel", "LlamaPreTrainedModel", + "LlamaForSequenceClassification", ] @@ -63,11 +64,7 @@ except OptionalDependencyNotAvailable: pass else: - from .modeling_llama import ( - LlamaForCausalLM, - LlamaModel, - LlamaPreTrainedModel, - ) + from .modeling_llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel else: diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index 40a3f0f802a0..5cdea1b787f6 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -24,19 +24,12 @@ import torch import torch.utils.checkpoint from torch import nn -from torch.nn import CrossEntropyLoss +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import ( - BaseModelOutputWithPast, - CausalLMOutputWithPast, -) +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast from ...modeling_utils import PreTrainedModel -from ...utils import ( - add_start_docstrings, - logging, - replace_return_docstrings, -) +from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings from .configuration_llama import LlamaConfig @@ -357,7 +350,7 @@ def forward( @add_start_docstrings( - "The bare OPT Model outputting raw hidden-states without any specific head on top.", + "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", LLAMA_START_DOCSTRING, ) class LlamaPreTrainedModel(PreTrainedModel): @@ -831,3 +824,122 @@ def _reorder_cache(past_key_values, beam_idx): for layer_past in past_key_values: reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) return reordered_past + + +@add_start_docstrings( + """ + The LLaMa Model transformer with a sequence classification head on top (linear layer). + + [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + LLAMA_START_DOCSTRING, +) +class LlamaForSequenceClassification(LlamaPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = LlamaModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + sequence_lengths = (torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1).to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(pooled_logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(pooled_logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(pooled_logits, labels) + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 6fb3d2d9d0bb..a80af49e2784 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -3768,6 +3768,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class LlamaForSequenceClassification(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class LlamaModel(metaclass=DummyObject): _backends = ["torch"] diff --git a/tests/models/llama/test_modeling_llama.py b/tests/models/llama/test_modeling_llama.py index 23d573454da4..dea92d5111fd 100644 --- a/tests/models/llama/test_modeling_llama.py +++ b/tests/models/llama/test_modeling_llama.py @@ -27,7 +27,7 @@ if is_torch_available(): import torch - from transformers import LlamaForCausalLM, LlamaModel + from transformers import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel class LlamaModelTester: @@ -255,14 +255,7 @@ def prepare_config_and_inputs_for_common(self): @require_torch class LlamaModelTest(ModelTesterMixin, unittest.TestCase): - all_model_classes = ( - ( - LlamaModel, - LlamaForCausalLM, - ) - if is_torch_available() - else () - ) + all_model_classes = (LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else () all_generative_model_classes = (LlamaForCausalLM,) if is_torch_available() else () test_headmasking = False @@ -283,6 +276,46 @@ def test_model_various_embeddings(self): config_and_inputs[0].position_embedding_type = type self.model_tester.create_and_check_model(*config_and_inputs) + def test_llama_sequence_classification_model(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.num_labels = 3 + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + sequence_labels = ids_tensor([self.model_tester.batch_size], self.model_tester.type_sequence_label_size) + model = LlamaForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + + def test_llama_sequence_classification_model_for_single_label(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.num_labels = 3 + config.problem_type = "single_label_classification" + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + sequence_labels = ids_tensor([self.model_tester.batch_size], self.model_tester.type_sequence_label_size) + model = LlamaForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + + def test_llama_sequence_classification_model_for_multi_label(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.num_labels = 3 + config.problem_type = "multi_label_classification" + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + sequence_labels = ids_tensor( + [self.model_tester.batch_size, config.num_labels], self.model_tester.type_sequence_label_size + ).to(torch.float) + model = LlamaForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + @unittest.skip("LLaMA does not support head pruning.") def test_head_pruning(self): pass From dae8b3b7f0e26270b02bcb83df8c62f029730d2f Mon Sep 17 00:00:00 2001 From: Maria Khalusova Date: Fri, 17 Mar 2023 10:27:12 -0400 Subject: [PATCH 126/277] Removed .mdx extension in two links (#22230) removed .mdx extension --- docs/source/en/model_doc/dpt.mdx | 2 +- docs/source/en/model_doc/glpn.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/dpt.mdx b/docs/source/en/model_doc/dpt.mdx index c80ab8db7c0e..030b73e6ddf1 100644 --- a/docs/source/en/model_doc/dpt.mdx +++ b/docs/source/en/model_doc/dpt.mdx @@ -34,7 +34,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - Demo notebooks for [`DPTForDepthEstimation`] can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/DPT). - [Semantic segmentation task guide](../tasks/semantic_segmentation) -- [Monocular depth estimation task guide](../tasks/monocular_depth_estimation.mdx) +- [Monocular depth estimation task guide](../tasks/monocular_depth_estimation) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/glpn.mdx b/docs/source/en/model_doc/glpn.mdx index ce930b2e95a2..d1ae7da094d5 100644 --- a/docs/source/en/model_doc/glpn.mdx +++ b/docs/source/en/model_doc/glpn.mdx @@ -45,7 +45,7 @@ This model was contributed by [nielsr](https://huggingface.co/nielsr). The origi A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with GLPN. - Demo notebooks for [`GLPNForDepthEstimation`] can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/GLPN). -- [Monocular depth estimation task guide](../tasks/monocular_depth_estimation.mdx) +- [Monocular depth estimation task guide](../tasks/monocular_depth_estimation) ## GLPNConfig From b923c059a755c06524a587ada295c9ba62d42f70 Mon Sep 17 00:00:00 2001 From: Seb0 Date: Fri, 17 Mar 2023 15:30:17 +0100 Subject: [PATCH 127/277] fix(docs): fix task guide links in model docs (#22226) fix(docs): task guide links in model docs --- docs/source/en/model_doc/albert.mdx | 10 +++++----- .../audio-spectrogram-transformer.mdx | 2 +- docs/source/en/model_doc/bart.mdx | 12 +++++------ docs/source/en/model_doc/beit.mdx | 4 ++-- docs/source/en/model_doc/bert.mdx | 10 +++++----- docs/source/en/model_doc/big_bird.mdx | 12 +++++------ docs/source/en/model_doc/bigbird_pegasus.mdx | 10 +++++----- docs/source/en/model_doc/biogpt.mdx | 2 +- docs/source/en/model_doc/bit.mdx | 2 +- docs/source/en/model_doc/blenderbot-small.mdx | 6 +++--- docs/source/en/model_doc/blenderbot.mdx | 6 +++--- docs/source/en/model_doc/bloom.mdx | 8 ++++---- docs/source/en/model_doc/camembert.mdx | 12 +++++------ docs/source/en/model_doc/canine.mdx | 8 ++++---- docs/source/en/model_doc/codegen.mdx | 2 +- docs/source/en/model_doc/conditional_detr.mdx | 2 +- docs/source/en/model_doc/convbert.mdx | 10 +++++----- docs/source/en/model_doc/convnext.mdx | 2 +- docs/source/en/model_doc/ctrl.mdx | 4 ++-- docs/source/en/model_doc/cvt.mdx | 2 +- docs/source/en/model_doc/data2vec.mdx | 20 +++++++++---------- docs/source/en/model_doc/deberta-v2.mdx | 10 +++++----- docs/source/en/model_doc/deberta.mdx | 8 ++++---- docs/source/en/model_doc/deformable_detr.mdx | 2 +- docs/source/en/model_doc/deit.mdx | 2 +- docs/source/en/model_doc/deta.mdx | 2 +- docs/source/en/model_doc/detr.mdx | 2 +- docs/source/en/model_doc/dinat.mdx | 2 +- docs/source/en/model_doc/distilbert.mdx | 10 +++++----- docs/source/en/model_doc/dpt.mdx | 1 + docs/source/en/model_doc/efficientformer.mdx | 2 +- docs/source/en/model_doc/electra.mdx | 12 +++++------ docs/source/en/model_doc/ernie.mdx | 12 +++++------ docs/source/en/model_doc/ernie_m.mdx | 8 ++++---- docs/source/en/model_doc/esm.mdx | 6 +++--- docs/source/en/model_doc/flaubert.mdx | 10 +++++----- docs/source/en/model_doc/fnet.mdx | 10 +++++----- docs/source/en/model_doc/funnel.mdx | 10 +++++----- docs/source/en/model_doc/git.mdx | 2 +- docs/source/en/model_doc/gpt-sw3.mdx | 6 +++--- docs/source/en/model_doc/gpt2.mdx | 6 +++--- docs/source/en/model_doc/gpt_neo.mdx | 4 ++-- docs/source/en/model_doc/gpt_neox.mdx | 2 +- .../source/en/model_doc/gpt_neox_japanese.mdx | 2 +- docs/source/en/model_doc/gptj.mdx | 6 +++--- docs/source/en/model_doc/hubert.mdx | 4 ++-- docs/source/en/model_doc/ibert.mdx | 10 +++++----- docs/source/en/model_doc/imagegpt.mdx | 2 +- docs/source/en/model_doc/layoutlm.mdx | 8 ++++---- docs/source/en/model_doc/layoutlmv2.mdx | 8 ++++---- docs/source/en/model_doc/layoutlmv3.mdx | 8 ++++---- docs/source/en/model_doc/led.mdx | 8 ++++---- docs/source/en/model_doc/levit.mdx | 2 +- docs/source/en/model_doc/lilt.mdx | 6 +++--- docs/source/en/model_doc/longformer.mdx | 10 +++++----- docs/source/en/model_doc/longt5.mdx | 4 ++-- docs/source/en/model_doc/luke.mdx | 10 +++++----- docs/source/en/model_doc/lxmert.mdx | 2 +- docs/source/en/model_doc/m2m_100.mdx | 4 ++-- docs/source/en/model_doc/marian.mdx | 6 +++--- docs/source/en/model_doc/markuplm.mdx | 6 +++--- docs/source/en/model_doc/mbart.mdx | 12 +++++------ docs/source/en/model_doc/mctct.mdx | 2 +- docs/source/en/model_doc/megatron-bert.mdx | 12 +++++------ docs/source/en/model_doc/mobilebert.mdx | 10 +++++----- docs/source/en/model_doc/mobilenet_v1.mdx | 2 +- docs/source/en/model_doc/mobilenet_v2.mdx | 4 ++-- docs/source/en/model_doc/mobilevit.mdx | 4 ++-- docs/source/en/model_doc/mpnet.mdx | 10 +++++----- docs/source/en/model_doc/mt5.mdx | 4 ++-- docs/source/en/model_doc/mvp.mdx | 12 +++++------ docs/source/en/model_doc/nat.mdx | 2 +- docs/source/en/model_doc/nezha.mdx | 10 +++++----- docs/source/en/model_doc/nllb.mdx | 4 ++-- docs/source/en/model_doc/nystromformer.mdx | 10 +++++----- docs/source/en/model_doc/openai-gpt.mdx | 4 ++-- docs/source/en/model_doc/pegasus.mdx | 6 +++--- docs/source/en/model_doc/pegasus_x.mdx | 4 ++-- docs/source/en/model_doc/perceiver.mdx | 6 +++--- docs/source/en/model_doc/plbart.mdx | 8 ++++---- docs/source/en/model_doc/poolformer.mdx | 2 +- docs/source/en/model_doc/prophetnet.mdx | 6 +++--- docs/source/en/model_doc/qdqbert.mdx | 12 +++++------ docs/source/en/model_doc/reformer.mdx | 8 ++++---- docs/source/en/model_doc/regnet.mdx | 2 +- docs/source/en/model_doc/rembert.mdx | 12 +++++------ docs/source/en/model_doc/resnet.mdx | 2 +- .../en/model_doc/roberta-prelayernorm.mdx | 12 +++++------ docs/source/en/model_doc/roberta.mdx | 10 +++++----- docs/source/en/model_doc/roc_bert.mdx | 12 +++++------ docs/source/en/model_doc/roformer.mdx | 12 +++++------ docs/source/en/model_doc/segformer.mdx | 4 ++-- docs/source/en/model_doc/sew-d.mdx | 4 ++-- docs/source/en/model_doc/sew.mdx | 4 ++-- docs/source/en/model_doc/speech_to_text_2.mdx | 2 +- docs/source/en/model_doc/splinter.mdx | 2 +- docs/source/en/model_doc/squeezebert.mdx | 10 +++++----- docs/source/en/model_doc/swin.mdx | 2 +- docs/source/en/model_doc/swinv2.mdx | 2 +- .../en/model_doc/switch_transformers.mdx | 4 ++-- docs/source/en/model_doc/t5.mdx | 4 ++-- docs/source/en/model_doc/tapas.mdx | 4 ++-- docs/source/en/model_doc/timesformer.mdx | 2 +- docs/source/en/model_doc/transfo-xl.mdx | 4 ++-- docs/source/en/model_doc/unispeech-sat.mdx | 4 ++-- docs/source/en/model_doc/unispeech.mdx | 4 ++-- docs/source/en/model_doc/upernet.mdx | 2 +- docs/source/en/model_doc/van.mdx | 2 +- docs/source/en/model_doc/videomae.mdx | 2 +- docs/source/en/model_doc/vit.mdx | 2 +- docs/source/en/model_doc/vit_hybrid.mdx | 2 +- docs/source/en/model_doc/vit_msn.mdx | 2 +- .../en/model_doc/wav2vec2-conformer.mdx | 4 ++-- docs/source/en/model_doc/wav2vec2.mdx | 4 ++-- docs/source/en/model_doc/wavlm.mdx | 4 ++-- docs/source/en/model_doc/xglm.mdx | 2 +- docs/source/en/model_doc/xlm-prophetnet.mdx | 6 +++--- docs/source/en/model_doc/xlm-roberta-xl.mdx | 12 +++++------ docs/source/en/model_doc/xlm-roberta.mdx | 12 +++++------ docs/source/en/model_doc/xlm.mdx | 12 +++++------ docs/source/en/model_doc/xlnet.mdx | 10 +++++----- docs/source/en/model_doc/xmod.mdx | 12 +++++------ docs/source/en/model_doc/yolos.mdx | 2 +- docs/source/en/model_doc/yoso.mdx | 10 +++++----- 124 files changed, 378 insertions(+), 377 deletions(-) diff --git a/docs/source/en/model_doc/albert.mdx b/docs/source/en/model_doc/albert.mdx index 1a8d6759c159..5076a5150be6 100644 --- a/docs/source/en/model_doc/albert.mdx +++ b/docs/source/en/model_doc/albert.mdx @@ -58,11 +58,11 @@ This model was contributed by [lysandre](https://huggingface.co/lysandre). This ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## AlbertConfig diff --git a/docs/source/en/model_doc/audio-spectrogram-transformer.mdx b/docs/source/en/model_doc/audio-spectrogram-transformer.mdx index e1572697e070..f4a8f71d9797 100644 --- a/docs/source/en/model_doc/audio-spectrogram-transformer.mdx +++ b/docs/source/en/model_doc/audio-spectrogram-transformer.mdx @@ -47,7 +47,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - A notebook illustrating inference with AST for audio classification can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/AST). - [`ASTForAudioClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/audio-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/audio_classification.ipynb). -- See also: [Audio classification](./tasks/audio_classification). +- See also: [Audio classification](../tasks/audio_classification). If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/bart.mdx b/docs/source/en/model_doc/bart.mdx index beb6153b63e9..ef4bebd01704 100644 --- a/docs/source/en/model_doc/bart.mdx +++ b/docs/source/en/model_doc/bart.mdx @@ -109,7 +109,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFBartForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/summarization) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization-tf.ipynb). - [`FlaxBartForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/summarization). - [Summarization](https://huggingface.co/course/chapter7/5?fw=pt#summarization) chapter of the 🤗 Hugging Face course. -- [Summarization task guide](./tasks/summarization) +- [Summarization task guide](../tasks/summarization) @@ -117,19 +117,19 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFBartForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_mlmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). - [`FlaxBartForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling#masked-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/masked_language_modeling_flax.ipynb). - [Masked language modeling](https://huggingface.co/course/chapter7/3?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) - A notebook on how to [finetune mBART using Seq2SeqTrainer for Hindi to English translation](https://colab.research.google.com/github/vasudevgupta7/huggingface-tutorials/blob/main/translation_training.ipynb). 🌎 - [`BartForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/translation) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/translation.ipynb). - [`TFBartForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/translation) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/translation-tf.ipynb). -- [Translation task guide](./tasks/translation) +- [Translation task guide](../tasks/translation) See also: -- [Text classification task guide](./tasks/sequence_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) ## BartConfig diff --git a/docs/source/en/model_doc/beit.mdx b/docs/source/en/model_doc/beit.mdx index ff423b3717c6..c9ca2ee9921d 100644 --- a/docs/source/en/model_doc/beit.mdx +++ b/docs/source/en/model_doc/beit.mdx @@ -74,10 +74,10 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`BeitForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) **Semantic segmentation** -- [Semantic segmentation task guide](./tasks/semantic_segmentation) +- [Semantic segmentation task guide](../tasks/semantic_segmentation) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/bert.mdx b/docs/source/en/model_doc/bert.mdx index d3b3ff217b9a..eae3bce130cf 100644 --- a/docs/source/en/model_doc/bert.mdx +++ b/docs/source/en/model_doc/bert.mdx @@ -72,7 +72,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`BertForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb). - [`TFBertForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification-tf.ipynb). - [`FlaxBertForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_flax.ipynb). -- [Text classification task guide](./tasks/sequence_classification) +- [Text classification task guide](../tasks/sequence_classification) @@ -82,7 +82,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFBertForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/token-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb). - [`FlaxBertForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/token-classification). - [Token classification](https://huggingface.co/course/chapter7/2?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Token classification task guide](./tasks/token_classification) +- [Token classification task guide](../tasks/token_classification) @@ -90,7 +90,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFBertForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_mlmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). - [`FlaxBertForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling#masked-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/masked_language_modeling_flax.ipynb). - [Masked language modeling](https://huggingface.co/course/chapter7/3?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) @@ -98,12 +98,12 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFBertForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/question-answering) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb). - [`FlaxBertForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/question-answering). - [Question answering](https://huggingface.co/course/chapter7/7?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Question answering task guide](./tasks/question_answering) +- [Question answering task guide](../tasks/question_answering) **Multiple choice** - [`BertForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb). - [`TFBertForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice-tf.ipynb). -- [Multiple choice task guide](./tasks/multiple_choice) +- [Multiple choice task guide](../tasks/multiple_choice) ⚡️ **Inference** - A blog post on how to [Accelerate BERT inference with Hugging Face Transformers and AWS Inferentia](https://huggingface.co/blog/bert-inferentia-sagemaker). diff --git a/docs/source/en/model_doc/big_bird.mdx b/docs/source/en/model_doc/big_bird.mdx index b0511aeb6c60..22bc4debf45e 100644 --- a/docs/source/en/model_doc/big_bird.mdx +++ b/docs/source/en/model_doc/big_bird.mdx @@ -54,12 +54,12 @@ This model was contributed by [vasudevgupta](https://huggingface.co/vasudevgupta ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## BigBirdConfig diff --git a/docs/source/en/model_doc/bigbird_pegasus.mdx b/docs/source/en/model_doc/bigbird_pegasus.mdx index 23e741f4ea1b..00ccb88dc0fa 100644 --- a/docs/source/en/model_doc/bigbird_pegasus.mdx +++ b/docs/source/en/model_doc/bigbird_pegasus.mdx @@ -54,11 +54,11 @@ The original code can be found [here](https://github.com/google-research/bigbird ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Text classification task guide](../tasks/sequence_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## BigBirdPegasusConfig diff --git a/docs/source/en/model_doc/biogpt.mdx b/docs/source/en/model_doc/biogpt.mdx index a2a17ad32dc5..e852c83a3645 100644 --- a/docs/source/en/model_doc/biogpt.mdx +++ b/docs/source/en/model_doc/biogpt.mdx @@ -31,7 +31,7 @@ This model was contributed by [kamalkraj](https://huggingface.co/kamalkraj). The ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) +- [Causal language modeling task guide](../tasks/language_modeling) ## BioGptConfig diff --git a/docs/source/en/model_doc/bit.mdx b/docs/source/en/model_doc/bit.mdx index 2f0cba77f549..343832e62e63 100644 --- a/docs/source/en/model_doc/bit.mdx +++ b/docs/source/en/model_doc/bit.mdx @@ -37,7 +37,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`BitForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/blenderbot-small.mdx b/docs/source/en/model_doc/blenderbot-small.mdx index c756346fea96..fa3d32ab0ec7 100644 --- a/docs/source/en/model_doc/blenderbot-small.mdx +++ b/docs/source/en/model_doc/blenderbot-small.mdx @@ -46,9 +46,9 @@ found [here](https://github.com/facebookresearch/ParlAI). ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## BlenderbotSmallConfig diff --git a/docs/source/en/model_doc/blenderbot.mdx b/docs/source/en/model_doc/blenderbot.mdx index 00de4288d25f..485b4ca1e0f6 100644 --- a/docs/source/en/model_doc/blenderbot.mdx +++ b/docs/source/en/model_doc/blenderbot.mdx @@ -68,9 +68,9 @@ Here is an example of model usage: ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## BlenderbotConfig diff --git a/docs/source/en/model_doc/bloom.mdx b/docs/source/en/model_doc/bloom.mdx index fd3df58e437c..3f30f14706a1 100644 --- a/docs/source/en/model_doc/bloom.mdx +++ b/docs/source/en/model_doc/bloom.mdx @@ -34,10 +34,10 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`BloomForCausalLM`] is supported by this [causal language modeling example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling#gpt-2gpt-and-causal-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb). See also: -- [Causal language modeling task guide](./tasks/language_modeling) -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) ⚡️ Inference diff --git a/docs/source/en/model_doc/camembert.mdx b/docs/source/en/model_doc/camembert.mdx index 78a2b8a5853e..4299603613c9 100644 --- a/docs/source/en/model_doc/camembert.mdx +++ b/docs/source/en/model_doc/camembert.mdx @@ -39,12 +39,12 @@ This model was contributed by [camembert](https://huggingface.co/camembert). The ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## CamembertConfig diff --git a/docs/source/en/model_doc/canine.mdx b/docs/source/en/model_doc/canine.mdx index 175f7c304eb3..b23da136d318 100644 --- a/docs/source/en/model_doc/canine.mdx +++ b/docs/source/en/model_doc/canine.mdx @@ -94,10 +94,10 @@ sequences to the same length): ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Multiple choice task guide](../tasks/multiple_choice) ## CANINE specific outputs diff --git a/docs/source/en/model_doc/codegen.mdx b/docs/source/en/model_doc/codegen.mdx index f36aec36a586..bc6de54b4e23 100644 --- a/docs/source/en/model_doc/codegen.mdx +++ b/docs/source/en/model_doc/codegen.mdx @@ -58,7 +58,7 @@ hello_world() ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) +- [Causal language modeling task guide](../tasks/language_modeling) ## CodeGenConfig diff --git a/docs/source/en/model_doc/conditional_detr.mdx b/docs/source/en/model_doc/conditional_detr.mdx index 4d51aa4cda06..0adb028908e9 100644 --- a/docs/source/en/model_doc/conditional_detr.mdx +++ b/docs/source/en/model_doc/conditional_detr.mdx @@ -29,7 +29,7 @@ This model was contributed by [DepuMeng](https://huggingface.co/DepuMeng). The o ## Documentation resources -- [Object detection task guide](./tasks/object_detection) +- [Object detection task guide](../tasks/object_detection) ## ConditionalDetrConfig diff --git a/docs/source/en/model_doc/convbert.mdx b/docs/source/en/model_doc/convbert.mdx index dbb0c3d420fb..cc2a79f9616f 100644 --- a/docs/source/en/model_doc/convbert.mdx +++ b/docs/source/en/model_doc/convbert.mdx @@ -47,11 +47,11 @@ here: https://github.com/yitu-opensource/ConvBert ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## ConvBertConfig diff --git a/docs/source/en/model_doc/convnext.mdx b/docs/source/en/model_doc/convnext.mdx index 389d01799237..3da76eb0e046 100644 --- a/docs/source/en/model_doc/convnext.mdx +++ b/docs/source/en/model_doc/convnext.mdx @@ -47,7 +47,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`ConvNextForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/ctrl.mdx b/docs/source/en/model_doc/ctrl.mdx index 54f21b6e4279..5b42fbe7c387 100644 --- a/docs/source/en/model_doc/ctrl.mdx +++ b/docs/source/en/model_doc/ctrl.mdx @@ -57,8 +57,8 @@ This model was contributed by [keskarnitishr](https://huggingface.co/keskarnitis ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Causal language modeling task guide](../tasks/language_modeling) ## CTRLConfig diff --git a/docs/source/en/model_doc/cvt.mdx b/docs/source/en/model_doc/cvt.mdx index 3b9fa2c3654e..3c58c5b7a2b5 100644 --- a/docs/source/en/model_doc/cvt.mdx +++ b/docs/source/en/model_doc/cvt.mdx @@ -45,7 +45,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`CvtForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/data2vec.mdx b/docs/source/en/model_doc/data2vec.mdx index c9df6d2d5e4a..091d34a163d6 100644 --- a/docs/source/en/model_doc/data2vec.mdx +++ b/docs/source/en/model_doc/data2vec.mdx @@ -55,20 +55,20 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - To fine-tune [`TFData2VecVisionForImageClassification`] on a custom dataset, see [this notebook](https://colab.research.google.com/github/sayakpaul/TF-2.0-Hacks/blob/master/data2vec_vision_image_classification.ipynb). **Data2VecText documentation resources** -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) **Data2VecAudio documentation resources** -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) **Data2VecVision documentation resources** -- [Image classification](./tasks/image_classification) -- [Semantic segmentation](./tasks/semantic_segmentation) +- [Image classification](../tasks/image_classification) +- [Semantic segmentation](../tasks/semantic_segmentation) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/deberta-v2.mdx b/docs/source/en/model_doc/deberta-v2.mdx index 63129a1ef7c8..74d537cf2243 100644 --- a/docs/source/en/model_doc/deberta-v2.mdx +++ b/docs/source/en/model_doc/deberta-v2.mdx @@ -60,11 +60,11 @@ contributed by [kamalkraj](https://huggingface.co/kamalkraj). The original code ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## DebertaV2Config diff --git a/docs/source/en/model_doc/deberta.mdx b/docs/source/en/model_doc/deberta.mdx index ec68ed03aec9..8faf2e1c28c1 100644 --- a/docs/source/en/model_doc/deberta.mdx +++ b/docs/source/en/model_doc/deberta.mdx @@ -48,7 +48,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - A blog post on [Supercharged Customer Service with Machine Learning](https://huggingface.co/blog/supercharge-customer-service-with-machine-learning) with DeBERTa. - [`DebertaForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb). - [`TFDebertaForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification-tf.ipynb). -- [Text classification task guide](./tasks/sequence_classification) +- [Text classification task guide](../tasks/sequence_classification) @@ -56,21 +56,21 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFDebertaForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/token-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb). - [Token classification](https://huggingface.co/course/chapter7/2?fw=pt) chapter of the 🤗 Hugging Face Course. - [Byte-Pair Encoding tokenization](https://huggingface.co/course/chapter6/5?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Token classification task guide](./tasks/token_classification) +- [Token classification task guide](../tasks/token_classification) - [`DebertaForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling#robertabertdistilbert-and-masked-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb). - [`TFDebertaForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_mlmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). - [Masked language modeling](https://huggingface.co/course/chapter7/3?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) - [`DebertaForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering.ipynb). - [`TFDebertaForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/question-answering) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb). - [Question answering](https://huggingface.co/course/chapter7/7?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Question answering task guide](./tasks/question_answering) +- [Question answering task guide](../tasks/question_answering) ## DebertaConfig diff --git a/docs/source/en/model_doc/deformable_detr.mdx b/docs/source/en/model_doc/deformable_detr.mdx index f9b763c7ba73..16ea70511b34 100644 --- a/docs/source/en/model_doc/deformable_detr.mdx +++ b/docs/source/en/model_doc/deformable_detr.mdx @@ -40,7 +40,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - Demo notebooks regarding inference + fine-tuning on a custom dataset for [`DeformableDetrForObjectDetection`] can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/Deformable-DETR). -- See also: [Object detection task guide](./tasks/object_detection). +- See also: [Object detection task guide](../tasks/object_detection). If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/deit.mdx b/docs/source/en/model_doc/deit.mdx index 4e38b1e2ad31..306915e7483e 100644 --- a/docs/source/en/model_doc/deit.mdx +++ b/docs/source/en/model_doc/deit.mdx @@ -78,7 +78,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`DeiTForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) Besides that: diff --git a/docs/source/en/model_doc/deta.mdx b/docs/source/en/model_doc/deta.mdx index d9a411d2eb7a..9b45e4475329 100644 --- a/docs/source/en/model_doc/deta.mdx +++ b/docs/source/en/model_doc/deta.mdx @@ -39,7 +39,7 @@ The original code can be found [here](https://github.com/jozhang97/DETA). A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with DETA. - Demo notebooks for DETA can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/DETA). -- See also: [Object detection task guide](./tasks/object_detection) +- See also: [Object detection task guide](../tasks/object_detection) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/detr.mdx b/docs/source/en/model_doc/detr.mdx index 2467a085fc46..57fbd04e1883 100644 --- a/docs/source/en/model_doc/detr.mdx +++ b/docs/source/en/model_doc/detr.mdx @@ -157,7 +157,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - All example notebooks illustrating fine-tuning [`DetrForObjectDetection`] and [`DetrForSegmentation`] on a custom dataset an be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/DETR). -- See also: [Object detection task guide](./tasks/object_detection) +- See also: [Object detection task guide](../tasks/object_detection) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/dinat.mdx b/docs/source/en/model_doc/dinat.mdx index bb56d22627b7..2ddbc63224c3 100644 --- a/docs/source/en/model_doc/dinat.mdx +++ b/docs/source/en/model_doc/dinat.mdx @@ -68,7 +68,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`DinatForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/distilbert.mdx b/docs/source/en/model_doc/distilbert.mdx index 43b329952cb7..cc1e03715118 100644 --- a/docs/source/en/model_doc/distilbert.mdx +++ b/docs/source/en/model_doc/distilbert.mdx @@ -75,7 +75,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`DistilBertForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb). - [`TFDistilBertForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification-tf.ipynb). - [`FlaxDistilBertForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_flax.ipynb). -- [Text classification task guide](./tasks/sequence_classification) +- [Text classification task guide](../tasks/sequence_classification) @@ -84,7 +84,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFDistilBertForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/token-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb). - [`FlaxDistilBertForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/token-classification). - [Token classification](https://huggingface.co/course/chapter7/2?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Token classification task guide](./tasks/token_classification) +- [Token classification task guide](../tasks/token_classification) @@ -93,7 +93,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFDistilBertForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_mlmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). - [`FlaxDistilBertForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling#masked-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/masked_language_modeling_flax.ipynb). - [Masked language modeling](https://huggingface.co/course/chapter7/3?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) @@ -101,12 +101,12 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFDistilBertForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/question-answering) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb). - [`FlaxDistilBertForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/question-answering). - [Question answering](https://huggingface.co/course/chapter7/7?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Question answering task guide](./tasks/question_answering) +- [Question answering task guide](../tasks/question_answering) **Multiple choice** - [`DistilBertForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb). - [`TFDistilBertForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice-tf.ipynb). -- [Multiple choice task guide](./tasks/multiple_choice) +- [Multiple choice task guide](../tasks/multiple_choice) ⚗️ Optimization diff --git a/docs/source/en/model_doc/dpt.mdx b/docs/source/en/model_doc/dpt.mdx index 030b73e6ddf1..e67dfde084d8 100644 --- a/docs/source/en/model_doc/dpt.mdx +++ b/docs/source/en/model_doc/dpt.mdx @@ -33,6 +33,7 @@ This model was contributed by [nielsr](https://huggingface.co/nielsr). The origi A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with DPT. - Demo notebooks for [`DPTForDepthEstimation`] can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/DPT). + - [Semantic segmentation task guide](../tasks/semantic_segmentation) - [Monocular depth estimation task guide](../tasks/monocular_depth_estimation) diff --git a/docs/source/en/model_doc/efficientformer.mdx b/docs/source/en/model_doc/efficientformer.mdx index e4b1b00701cb..2b512932ef82 100644 --- a/docs/source/en/model_doc/efficientformer.mdx +++ b/docs/source/en/model_doc/efficientformer.mdx @@ -41,7 +41,7 @@ The original code can be found [here](https://github.com/snap-research/Efficient ## Documentation resources -- [Image classification task guide](./tasks/image_classification) +- [Image classification task guide](../tasks/image_classification) ## EfficientFormerConfig diff --git a/docs/source/en/model_doc/electra.mdx b/docs/source/en/model_doc/electra.mdx index ce1c55487f96..e0422dbae5a8 100644 --- a/docs/source/en/model_doc/electra.mdx +++ b/docs/source/en/model_doc/electra.mdx @@ -66,12 +66,12 @@ This model was contributed by [lysandre](https://huggingface.co/lysandre). The o ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## ElectraConfig diff --git a/docs/source/en/model_doc/ernie.mdx b/docs/source/en/model_doc/ernie.mdx index 17869e2921a0..0159c0f4bc6a 100644 --- a/docs/source/en/model_doc/ernie.mdx +++ b/docs/source/en/model_doc/ernie.mdx @@ -49,12 +49,12 @@ and [ERNIE](https://github.com/PaddlePaddle/ERNIE/blob/repro). ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## ErnieConfig diff --git a/docs/source/en/model_doc/ernie_m.mdx b/docs/source/en/model_doc/ernie_m.mdx index 8861647e89a0..bd04483fc5b5 100644 --- a/docs/source/en/model_doc/ernie_m.mdx +++ b/docs/source/en/model_doc/ernie_m.mdx @@ -34,10 +34,10 @@ This model was contributed by [Susnato Dhar](https://huggingface.co/susnato). Th ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Multiple choice task guide](../tasks/multiple_choice) ## ErnieMConfig diff --git a/docs/source/en/model_doc/esm.mdx b/docs/source/en/model_doc/esm.mdx index 90e18ce45e53..ca7c950dabe0 100644 --- a/docs/source/en/model_doc/esm.mdx +++ b/docs/source/en/model_doc/esm.mdx @@ -88,9 +88,9 @@ The `openfold` library is licensed under the Apache License 2.0. ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Masked language modeling task guide](../tasks/masked_language_modeling) ## EsmConfig diff --git a/docs/source/en/model_doc/flaubert.mdx b/docs/source/en/model_doc/flaubert.mdx index 443916ee0781..0e8a1b252ab7 100644 --- a/docs/source/en/model_doc/flaubert.mdx +++ b/docs/source/en/model_doc/flaubert.mdx @@ -48,11 +48,11 @@ Tips: ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## FlaubertConfig diff --git a/docs/source/en/model_doc/fnet.mdx b/docs/source/en/model_doc/fnet.mdx index 140e61d4c4e2..9bf858bee427 100644 --- a/docs/source/en/model_doc/fnet.mdx +++ b/docs/source/en/model_doc/fnet.mdx @@ -43,11 +43,11 @@ This model was contributed by [gchhablani](https://huggingface.co/gchhablani). T ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## FNetConfig diff --git a/docs/source/en/model_doc/funnel.mdx b/docs/source/en/model_doc/funnel.mdx index 1edba7d7370e..ebbc67927872 100644 --- a/docs/source/en/model_doc/funnel.mdx +++ b/docs/source/en/model_doc/funnel.mdx @@ -62,11 +62,11 @@ This model was contributed by [sgugger](https://huggingface.co/sgugger). The ori ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## FunnelConfig diff --git a/docs/source/en/model_doc/git.mdx b/docs/source/en/model_doc/git.mdx index 0918e7c36c13..ac13874d472f 100644 --- a/docs/source/en/model_doc/git.mdx +++ b/docs/source/en/model_doc/git.mdx @@ -41,7 +41,7 @@ The original code can be found [here](https://github.com/microsoft/GenerativeIma A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with GIT. - Demo notebooks regarding inference + fine-tuning GIT on custom data can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/GIT). -- See also: [Causal language modeling task guide](./tasks/language_modeling) +- See also: [Causal language modeling task guide](../tasks/language_modeling) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we will review it. The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/gpt-sw3.mdx b/docs/source/en/model_doc/gpt-sw3.mdx index 0ee4a43f4c7b..2f20937a4c1e 100644 --- a/docs/source/en/model_doc/gpt-sw3.mdx +++ b/docs/source/en/model_doc/gpt-sw3.mdx @@ -50,9 +50,9 @@ Träd är fina för att de är färgstarka. Men ibland är det fint ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Causal language modeling task guide](../tasks/language_modeling) ## GPTSw3Tokenizer diff --git a/docs/source/en/model_doc/gpt2.mdx b/docs/source/en/model_doc/gpt2.mdx index d605a1449d20..2a8e693b4cac 100644 --- a/docs/source/en/model_doc/gpt2.mdx +++ b/docs/source/en/model_doc/gpt2.mdx @@ -73,9 +73,9 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`GPT2LMHeadModel`] is supported by this [causal language modeling example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling#gpt-2gpt-and-causal-language-modeling), [text generation example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/text-generation), and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb). - [`TFGPT2LMHeadModel`] is supported by this [causal language modeling example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_clmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). - [`FlaxGPT2LMHeadModel`] is supported by this [causal language modeling example script](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling#causal-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/causal_language_modeling_flax.ipynb). -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Causal language modeling task guide](../tasks/language_modeling) ## GPT2Config diff --git a/docs/source/en/model_doc/gpt_neo.mdx b/docs/source/en/model_doc/gpt_neo.mdx index af55c88a265a..a21a3bd87b32 100644 --- a/docs/source/en/model_doc/gpt_neo.mdx +++ b/docs/source/en/model_doc/gpt_neo.mdx @@ -52,8 +52,8 @@ The `generate()` method can be used to generate text using GPT Neo model. ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Causal language modeling task guide](../tasks/language_modeling) ## GPTNeoConfig diff --git a/docs/source/en/model_doc/gpt_neox.mdx b/docs/source/en/model_doc/gpt_neox.mdx index cd85f03e7374..1f5ec5e794b8 100644 --- a/docs/source/en/model_doc/gpt_neox.mdx +++ b/docs/source/en/model_doc/gpt_neox.mdx @@ -59,7 +59,7 @@ The `generate()` method can be used to generate text using GPT Neo model. ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) +- [Causal language modeling task guide](../tasks/language_modeling) ## GPTNeoXConfig diff --git a/docs/source/en/model_doc/gpt_neox_japanese.mdx b/docs/source/en/model_doc/gpt_neox_japanese.mdx index bc0c259f7403..d4bd3276e07c 100644 --- a/docs/source/en/model_doc/gpt_neox_japanese.mdx +++ b/docs/source/en/model_doc/gpt_neox_japanese.mdx @@ -49,7 +49,7 @@ The `generate()` method can be used to generate text using GPT NeoX Japanese mod ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) +- [Causal language modeling task guide](../tasks/language_modeling) ## GPTNeoXJapaneseConfig diff --git a/docs/source/en/model_doc/gptj.mdx b/docs/source/en/model_doc/gptj.mdx index 2b407d7b2d40..b97251332670 100644 --- a/docs/source/en/model_doc/gptj.mdx +++ b/docs/source/en/model_doc/gptj.mdx @@ -125,9 +125,9 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`FlaxGPTJForCausalLM`] is supported by this [causal language modeling example script](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling#causal-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/causal_language_modeling_flax.ipynb). **Documentation resources** -- [Text classification task guide](./tasks/sequence_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) ## GPTJConfig diff --git a/docs/source/en/model_doc/hubert.mdx b/docs/source/en/model_doc/hubert.mdx index 55935e1a5c9a..0fcf93939544 100644 --- a/docs/source/en/model_doc/hubert.mdx +++ b/docs/source/en/model_doc/hubert.mdx @@ -42,8 +42,8 @@ This model was contributed by [patrickvonplaten](https://huggingface.co/patrickv ## Documentation resources -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) ## HubertConfig diff --git a/docs/source/en/model_doc/ibert.mdx b/docs/source/en/model_doc/ibert.mdx index f665374781d1..90735832c83e 100644 --- a/docs/source/en/model_doc/ibert.mdx +++ b/docs/source/en/model_doc/ibert.mdx @@ -38,11 +38,11 @@ This model was contributed by [kssteven](https://huggingface.co/kssteven). The o ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/masked_language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/masked_language_modeling) ## IBertConfig diff --git a/docs/source/en/model_doc/imagegpt.mdx b/docs/source/en/model_doc/imagegpt.mdx index 1c265922369a..13b03560b834 100644 --- a/docs/source/en/model_doc/imagegpt.mdx +++ b/docs/source/en/model_doc/imagegpt.mdx @@ -77,7 +77,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - Demo notebooks for ImageGPT can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/ImageGPT). - [`ImageGPTForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/layoutlm.mdx b/docs/source/en/model_doc/layoutlm.mdx index c33f59c3bd8c..f9e34922f6c9 100644 --- a/docs/source/en/model_doc/layoutlm.mdx +++ b/docs/source/en/model_doc/layoutlm.mdx @@ -88,20 +88,20 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - A notebook on how to [fine-tune LayoutLM on the FUNSD dataset with image embeddings](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Add_image_embeddings_to_LayoutLM.ipynb). -- See also: [Document question answering task guide](./tasks/document_question_answering) +- See also: [Document question answering task guide](../tasks/document_question_answering) - A notebook on how to [fine-tune LayoutLM for sequence classification on the RVL-CDIP dataset](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForSequenceClassification_on_RVL_CDIP.ipynb). -- [Text classification task guide](./tasks/sequence_classification) +- [Text classification task guide](../tasks/sequence_classification) - A notebook on how to [ fine-tune LayoutLM for token classification on the FUNSD dataset](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb). -- [Token classification task guide](./tasks/token_classification) +- [Token classification task guide](../tasks/token_classification) **Other resources** -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) 🚀 Deploy diff --git a/docs/source/en/model_doc/layoutlmv2.mdx b/docs/source/en/model_doc/layoutlmv2.mdx index af29db0574d6..031cce83deb2 100644 --- a/docs/source/en/model_doc/layoutlmv2.mdx +++ b/docs/source/en/model_doc/layoutlmv2.mdx @@ -268,10 +268,10 @@ print(encoding.keys()) ## Documentation resources -- [Document question answering task guide](./tasks/document_question_answering) -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) +- [Document question answering task guide](../tasks/document_question_answering) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) ## LayoutLMv2Config diff --git a/docs/source/en/model_doc/layoutlmv3.mdx b/docs/source/en/model_doc/layoutlmv3.mdx index 5145452dd852..2a8299a41334 100644 --- a/docs/source/en/model_doc/layoutlmv3.mdx +++ b/docs/source/en/model_doc/layoutlmv3.mdx @@ -52,22 +52,22 @@ LayoutLMv3 is nearly identical to LayoutLMv2, so we've also included LayoutLMv2 - [`LayoutLMv2ForSequenceClassification`] is supported by this [notebook](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLMv2/RVL-CDIP/Fine_tuning_LayoutLMv2ForSequenceClassification_on_RVL_CDIP.ipynb). -- [Text classification task guide](./tasks/sequence_classification) +- [Text classification task guide](../tasks/sequence_classification) - [`LayoutLMv3ForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/research_projects/layoutlmv3) and [notebook](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLMv3/Fine_tune_LayoutLMv3_on_FUNSD_(HuggingFace_Trainer).ipynb). - A [notebook](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLMv2/FUNSD/Inference_with_LayoutLMv2ForTokenClassification.ipynb) for how to perform inference with [`LayoutLMv2ForTokenClassification`] and a [notebook](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLMv2/FUNSD/True_inference_with_LayoutLMv2ForTokenClassification_%2B_Gradio_demo.ipynb) for how to perform inference when no labels are available with [`LayoutLMv2ForTokenClassification`]. - A [notebook](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLMv2/FUNSD/Fine_tuning_LayoutLMv2ForTokenClassification_on_FUNSD_using_HuggingFace_Trainer.ipynb) for how to finetune [`LayoutLMv2ForTokenClassification`] with the 🤗 Trainer. -- [Token classification task guide](./tasks/token_classification) +- [Token classification task guide](../tasks/token_classification) - [`LayoutLMv2ForQuestionAnswering`] is supported by this [notebook](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLMv2/DocVQA/Fine_tuning_LayoutLMv2ForQuestionAnswering_on_DocVQA.ipynb). -- [Question answering task guide](./tasks/question_answering) +- [Question answering task guide](../tasks/question_answering) **Document question answering** -- [Document question answering task guide](./tasks/document_question_answering) +- [Document question answering task guide](../tasks/document_question_answering) ## LayoutLMv3Config diff --git a/docs/source/en/model_doc/led.mdx b/docs/source/en/model_doc/led.mdx index a8916f68330b..537cfba52896 100644 --- a/docs/source/en/model_doc/led.mdx +++ b/docs/source/en/model_doc/led.mdx @@ -57,10 +57,10 @@ This model was contributed by [patrickvonplaten](https://huggingface.co/patrickv ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Question answering task guide](./tasks/question_answering) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Text classification task guide](../tasks/sequence_classification) +- [Question answering task guide](../tasks/question_answering) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## LEDConfig diff --git a/docs/source/en/model_doc/levit.mdx b/docs/source/en/model_doc/levit.mdx index 975c67a2379d..bfca2a5c063b 100644 --- a/docs/source/en/model_doc/levit.mdx +++ b/docs/source/en/model_doc/levit.mdx @@ -68,7 +68,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`LevitForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/lilt.mdx b/docs/source/en/model_doc/lilt.mdx index f238e6b3e99b..421bc066d1b2 100644 --- a/docs/source/en/model_doc/lilt.mdx +++ b/docs/source/en/model_doc/lilt.mdx @@ -53,9 +53,9 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - Demo notebooks for LiLT can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/LiLT). **Documentation resources** -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/longformer.mdx b/docs/source/en/model_doc/longformer.mdx index d725b96fe1b9..8170992624b2 100644 --- a/docs/source/en/model_doc/longformer.mdx +++ b/docs/source/en/model_doc/longformer.mdx @@ -91,11 +91,11 @@ loss = model(input_ids, labels=input_ids, masked_lm_labels=mlm_labels)[0] ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## LongformerConfig diff --git a/docs/source/en/model_doc/longt5.mdx b/docs/source/en/model_doc/longt5.mdx index ae771f44487c..9ae3d29a21e0 100644 --- a/docs/source/en/model_doc/longt5.mdx +++ b/docs/source/en/model_doc/longt5.mdx @@ -88,8 +88,8 @@ The original code can be found [here](https://github.com/google-research/longt5) ## Documentation resources -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## LongT5Config diff --git a/docs/source/en/model_doc/luke.mdx b/docs/source/en/model_doc/luke.mdx index 98e5501399dd..c96c098eb029 100644 --- a/docs/source/en/model_doc/luke.mdx +++ b/docs/source/en/model_doc/luke.mdx @@ -119,11 +119,11 @@ This model was contributed by [ikuyamada](https://huggingface.co/ikuyamada) and ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## LukeConfig diff --git a/docs/source/en/model_doc/lxmert.mdx b/docs/source/en/model_doc/lxmert.mdx index 6941dd99bbcd..083a98c34ceb 100644 --- a/docs/source/en/model_doc/lxmert.mdx +++ b/docs/source/en/model_doc/lxmert.mdx @@ -53,7 +53,7 @@ This model was contributed by [eltoto1219](https://huggingface.co/eltoto1219). T ## Documentation resources -- [Question answering task guide](./tasks/question_answering) +- [Question answering task guide](../tasks/question_answering) ## LxmertConfig diff --git a/docs/source/en/model_doc/m2m_100.mdx b/docs/source/en/model_doc/m2m_100.mdx index 2d62a556c00e..ebe49d578066 100644 --- a/docs/source/en/model_doc/m2m_100.mdx +++ b/docs/source/en/model_doc/m2m_100.mdx @@ -93,8 +93,8 @@ loss = model(**model_inputs).loss # forward pass ## Documentation resources -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## M2M100Config diff --git a/docs/source/en/model_doc/marian.mdx b/docs/source/en/model_doc/marian.mdx index 9192aa9080ce..dbca456978d0 100644 --- a/docs/source/en/model_doc/marian.mdx +++ b/docs/source/en/model_doc/marian.mdx @@ -163,9 +163,9 @@ Example of translating english to many romance languages, using old-style 2 char ## Documentation resources -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) +- [Causal language modeling task guide](../tasks/language_modeling) ## MarianConfig diff --git a/docs/source/en/model_doc/markuplm.mdx b/docs/source/en/model_doc/markuplm.mdx index 757eed974928..ef8d52ab255a 100644 --- a/docs/source/en/model_doc/markuplm.mdx +++ b/docs/source/en/model_doc/markuplm.mdx @@ -195,9 +195,9 @@ dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'xpath_tags_seq', 'x ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) ## MarkupLMConfig diff --git a/docs/source/en/model_doc/mbart.mdx b/docs/source/en/model_doc/mbart.mdx index 773b52a43658..ed9f1f5cfdc8 100644 --- a/docs/source/en/model_doc/mbart.mdx +++ b/docs/source/en/model_doc/mbart.mdx @@ -154,12 +154,12 @@ tokenizer.batch_decode(generated_tokens, skip_special_tokens=True) ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Text classification task guide](../tasks/sequence_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## MBartConfig diff --git a/docs/source/en/model_doc/mctct.mdx b/docs/source/en/model_doc/mctct.mdx index 46f07c763f12..89952a82fb97 100644 --- a/docs/source/en/model_doc/mctct.mdx +++ b/docs/source/en/model_doc/mctct.mdx @@ -33,7 +33,7 @@ This model was contributed by [cwkeam](https://huggingface.co/cwkeam). The origi ## Documentation resources -- [Automatic speech recognition task guide](./tasks/asr) +- [Automatic speech recognition task guide](../tasks/asr) Tips: diff --git a/docs/source/en/model_doc/megatron-bert.mdx b/docs/source/en/model_doc/megatron-bert.mdx index b3e09de9b42c..9fe5db6c7a89 100644 --- a/docs/source/en/model_doc/megatron-bert.mdx +++ b/docs/source/en/model_doc/megatron-bert.mdx @@ -80,12 +80,12 @@ Megatron Language models. In particular, it contains a hybrid model parallel app ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## MegatronBertConfig diff --git a/docs/source/en/model_doc/mobilebert.mdx b/docs/source/en/model_doc/mobilebert.mdx index 7fb2c7c4c4ed..9f6d6c6af011 100644 --- a/docs/source/en/model_doc/mobilebert.mdx +++ b/docs/source/en/model_doc/mobilebert.mdx @@ -45,11 +45,11 @@ This model was contributed by [vshampor](https://huggingface.co/vshampor). The o ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## MobileBertConfig diff --git a/docs/source/en/model_doc/mobilenet_v1.mdx b/docs/source/en/model_doc/mobilenet_v1.mdx index 902040f20812..6a9fff2765f4 100644 --- a/docs/source/en/model_doc/mobilenet_v1.mdx +++ b/docs/source/en/model_doc/mobilenet_v1.mdx @@ -51,7 +51,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`MobileNetV1ForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/mobilenet_v2.mdx b/docs/source/en/model_doc/mobilenet_v2.mdx index 356a9cb8d416..d4d6856a9122 100644 --- a/docs/source/en/model_doc/mobilenet_v2.mdx +++ b/docs/source/en/model_doc/mobilenet_v2.mdx @@ -55,10 +55,10 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`MobileNetV2ForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) **Semantic segmentation** -- [Semantic segmentation task guide](./tasks/semantic_segmentation) +- [Semantic segmentation task guide](../tasks/semantic_segmentation) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/mobilevit.mdx b/docs/source/en/model_doc/mobilevit.mdx index 1d627abf9566..c10069714546 100644 --- a/docs/source/en/model_doc/mobilevit.mdx +++ b/docs/source/en/model_doc/mobilevit.mdx @@ -64,10 +64,10 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`MobileViTForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) **Semantic segmentation** -- [Semantic segmentation task guide](./tasks/semantic_segmentation) +- [Semantic segmentation task guide](../tasks/semantic_segmentation) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/mpnet.mdx b/docs/source/en/model_doc/mpnet.mdx index 134dbef7e1a8..cf8b4de7a633 100644 --- a/docs/source/en/model_doc/mpnet.mdx +++ b/docs/source/en/model_doc/mpnet.mdx @@ -42,11 +42,11 @@ The original code can be found [here](https://github.com/microsoft/MPNet). ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## MPNetConfig diff --git a/docs/source/en/model_doc/mt5.mdx b/docs/source/en/model_doc/mt5.mdx index 93fac60bbad9..9de91982ac05 100644 --- a/docs/source/en/model_doc/mt5.mdx +++ b/docs/source/en/model_doc/mt5.mdx @@ -58,8 +58,8 @@ found [here](https://github.com/google-research/multilingual-t5). ## Documentation resources -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## MT5Config diff --git a/docs/source/en/model_doc/mvp.mdx b/docs/source/en/model_doc/mvp.mdx index 8cff270c0177..d457212e6cc1 100644 --- a/docs/source/en/model_doc/mvp.mdx +++ b/docs/source/en/model_doc/mvp.mdx @@ -102,12 +102,12 @@ For lightweight tuning, *i.e.*, fixing the model and only tuning prompts, you ca ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Text classification task guide](../tasks/sequence_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## MvpConfig diff --git a/docs/source/en/model_doc/nat.mdx b/docs/source/en/model_doc/nat.mdx index 25ac86368a61..5487dbee0c92 100644 --- a/docs/source/en/model_doc/nat.mdx +++ b/docs/source/en/model_doc/nat.mdx @@ -63,7 +63,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`NatForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/nezha.mdx b/docs/source/en/model_doc/nezha.mdx index 557e03113d09..3d6baf5d8a09 100644 --- a/docs/source/en/model_doc/nezha.mdx +++ b/docs/source/en/model_doc/nezha.mdx @@ -33,11 +33,11 @@ This model was contributed by [sijunhe](https://huggingface.co/sijunhe). The ori ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## NezhaConfig diff --git a/docs/source/en/model_doc/nllb.mdx b/docs/source/en/model_doc/nllb.mdx index 6b935a89aa7b..a53f718894af 100644 --- a/docs/source/en/model_doc/nllb.mdx +++ b/docs/source/en/model_doc/nllb.mdx @@ -90,8 +90,8 @@ UN-Chef sagt, es gibt keine militärische Lösung in Syrien ## Documentation resources -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## NllbTokenizer diff --git a/docs/source/en/model_doc/nystromformer.mdx b/docs/source/en/model_doc/nystromformer.mdx index 2c78892d8d54..1e6d1b212150 100644 --- a/docs/source/en/model_doc/nystromformer.mdx +++ b/docs/source/en/model_doc/nystromformer.mdx @@ -35,11 +35,11 @@ This model was contributed by [novice03](https://huggingface.co/novice03). The o ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## NystromformerConfig diff --git a/docs/source/en/model_doc/openai-gpt.mdx b/docs/source/en/model_doc/openai-gpt.mdx index 3adb22f0ab81..5c1b935ea9b7 100644 --- a/docs/source/en/model_doc/openai-gpt.mdx +++ b/docs/source/en/model_doc/openai-gpt.mdx @@ -73,7 +73,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - A blog post on [outperforming OpenAI GPT-3 with SetFit for text-classification](https://www.philschmid.de/getting-started-setfit). -- See also: [Text classification task guide](./tasks/sequence_classification) +- See also: [Text classification task guide](../tasks/sequence_classification) @@ -87,7 +87,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [Causal language modeling](https://huggingface.co/course/en/chapter7/6?fw=pt#training-a-causal-language-model-from-scratch) chapter of the 🤗 Hugging Face Course. - [`OpenAIGPTLMHeadModel`] is supported by this [causal language modeling example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling#gpt-2gpt-and-causal-language-modeling), [text generation example script](https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-generation/run_generation.py) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb). - [`TFOpenAIGPTLMHeadModel`] is supported by this [causal language modeling example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_clmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). -- See also: [Causal language modeling task guide](./tasks/language_modeling) +- See also: [Causal language modeling task guide](../tasks/language_modeling) diff --git a/docs/source/en/model_doc/pegasus.mdx b/docs/source/en/model_doc/pegasus.mdx index fe00ab203e60..41919a7e3c7f 100644 --- a/docs/source/en/model_doc/pegasus.mdx +++ b/docs/source/en/model_doc/pegasus.mdx @@ -104,9 +104,9 @@ All the [checkpoints](https://huggingface.co/models?search=pegasus) are fine-tun ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## PegasusConfig diff --git a/docs/source/en/model_doc/pegasus_x.mdx b/docs/source/en/model_doc/pegasus_x.mdx index f9aeb4643843..12e579a0545f 100644 --- a/docs/source/en/model_doc/pegasus_x.mdx +++ b/docs/source/en/model_doc/pegasus_x.mdx @@ -30,8 +30,8 @@ This model was contributed by [zphang]( - [`PoolFormerForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/prophetnet.mdx b/docs/source/en/model_doc/prophetnet.mdx index c49c659a28a1..071bdfbadf8c 100644 --- a/docs/source/en/model_doc/prophetnet.mdx +++ b/docs/source/en/model_doc/prophetnet.mdx @@ -55,9 +55,9 @@ The Authors' code can be found [here](https://github.com/microsoft/ProphetNet). ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## ProphetNetConfig diff --git a/docs/source/en/model_doc/qdqbert.mdx b/docs/source/en/model_doc/qdqbert.mdx index 76c35b60c4f0..3a65bd26c3a2 100644 --- a/docs/source/en/model_doc/qdqbert.mdx +++ b/docs/source/en/model_doc/qdqbert.mdx @@ -116,12 +116,12 @@ the instructions in [torch.onnx](https://pytorch.org/docs/stable/onnx.html). Exa ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## QDQBertConfig diff --git a/docs/source/en/model_doc/reformer.mdx b/docs/source/en/model_doc/reformer.mdx index b53eaa4e74ed..6725173d54da 100644 --- a/docs/source/en/model_doc/reformer.mdx +++ b/docs/source/en/model_doc/reformer.mdx @@ -153,10 +153,10 @@ loss = model(input_ids, labels=input_ids)[0] ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) ## ReformerConfig diff --git a/docs/source/en/model_doc/regnet.mdx b/docs/source/en/model_doc/regnet.mdx index e93eec6216d0..1557ab24df18 100644 --- a/docs/source/en/model_doc/regnet.mdx +++ b/docs/source/en/model_doc/regnet.mdx @@ -38,7 +38,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`RegNetForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/rembert.mdx b/docs/source/en/model_doc/rembert.mdx index fa3c16f90d8c..2a17a9b38470 100644 --- a/docs/source/en/model_doc/rembert.mdx +++ b/docs/source/en/model_doc/rembert.mdx @@ -39,12 +39,12 @@ also similar to the Albert one rather than the BERT one. ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## RemBertConfig diff --git a/docs/source/en/model_doc/resnet.mdx b/docs/source/en/model_doc/resnet.mdx index fd954967e580..476698e9ab9f 100644 --- a/docs/source/en/model_doc/resnet.mdx +++ b/docs/source/en/model_doc/resnet.mdx @@ -40,7 +40,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`ResNetForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/roberta-prelayernorm.mdx b/docs/source/en/model_doc/roberta-prelayernorm.mdx index 2a2d01c1c7b3..41b0762de761 100644 --- a/docs/source/en/model_doc/roberta-prelayernorm.mdx +++ b/docs/source/en/model_doc/roberta-prelayernorm.mdx @@ -31,12 +31,12 @@ The original code can be found [here](https://github.com/princeton-nlp/DinkyTrai ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## RobertaPreLayerNormConfig diff --git a/docs/source/en/model_doc/roberta.mdx b/docs/source/en/model_doc/roberta.mdx index 622853ab8936..7c0818a0144d 100644 --- a/docs/source/en/model_doc/roberta.mdx +++ b/docs/source/en/model_doc/roberta.mdx @@ -70,7 +70,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`RobertaForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification.ipynb). - [`TFRobertaForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification-tf.ipynb). - [`FlaxRobertaForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_flax.ipynb). -- [Text classification task guide](./tasks/sequence_classification) +- [Text classification task guide](../tasks/sequence_classification) @@ -78,7 +78,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFRobertaForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/token-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb). - [`FlaxRobertaForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/token-classification). - [Token classification](https://huggingface.co/course/chapter7/2?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Token classification task guide](./tasks/token_classification) +- [Token classification task guide](../tasks/token_classification) @@ -87,7 +87,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFRobertaForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_mlmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). - [`FlaxRobertaForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling#masked-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/masked_language_modeling_flax.ipynb). - [Masked language modeling](https://huggingface.co/course/chapter7/3?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) @@ -96,12 +96,12 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFRobertaForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/question-answering) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb). - [`FlaxRobertaForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/question-answering). - [Question answering](https://huggingface.co/course/chapter7/7?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Question answering task guide](./tasks/question_answering) +- [Question answering task guide](../tasks/question_answering) **Multiple choice** - [`RobertaForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb). - [`TFRobertaForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice-tf.ipynb). -- [Multiple choice task guide](./tasks/multiple_choice) +- [Multiple choice task guide](../tasks/multiple_choice) ## RobertaConfig diff --git a/docs/source/en/model_doc/roc_bert.mdx b/docs/source/en/model_doc/roc_bert.mdx index 42ff072e7448..50d123261e95 100644 --- a/docs/source/en/model_doc/roc_bert.mdx +++ b/docs/source/en/model_doc/roc_bert.mdx @@ -33,12 +33,12 @@ This model was contributed by [weiweishi](https://huggingface.co/weiweishi). ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## RoCBertConfig diff --git a/docs/source/en/model_doc/roformer.mdx b/docs/source/en/model_doc/roformer.mdx index 7940a5e24f27..942ca9ec5345 100644 --- a/docs/source/en/model_doc/roformer.mdx +++ b/docs/source/en/model_doc/roformer.mdx @@ -39,12 +39,12 @@ This model was contributed by [junnyu](https://huggingface.co/junnyu). The origi ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## RoFormerConfig diff --git a/docs/source/en/model_doc/segformer.mdx b/docs/source/en/model_doc/segformer.mdx index a5295789470c..1734a201958b 100644 --- a/docs/source/en/model_doc/segformer.mdx +++ b/docs/source/en/model_doc/segformer.mdx @@ -91,7 +91,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`SegformerForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- [Image classification task guide](./tasks/image_classification) +- [Image classification task guide](../tasks/image_classification) Semantic segmentation: @@ -99,7 +99,7 @@ Semantic segmentation: - A blog on fine-tuning SegFormer on a custom dataset can be found [here](https://huggingface.co/blog/fine-tune-segformer). - More demo notebooks on SegFormer (both inference + fine-tuning on a custom dataset) can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/SegFormer). - [`TFSegformerForSemanticSegmentation`] is supported by this [example notebook](https://github.com/huggingface/notebooks/blob/main/examples/semantic_segmentation-tf.ipynb). -- [Semantic segmentation task guide](./tasks/semantic_segmentation) +- [Semantic segmentation task guide](../tasks/semantic_segmentation) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/sew-d.mdx b/docs/source/en/model_doc/sew-d.mdx index c28c08773e93..1141be68570c 100644 --- a/docs/source/en/model_doc/sew-d.mdx +++ b/docs/source/en/model_doc/sew-d.mdx @@ -38,8 +38,8 @@ This model was contributed by [anton-l](https://huggingface.co/anton-l). ## Documentation resources -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) ## SEWDConfig diff --git a/docs/source/en/model_doc/sew.mdx b/docs/source/en/model_doc/sew.mdx index d37393f7320a..a4a780e992a2 100644 --- a/docs/source/en/model_doc/sew.mdx +++ b/docs/source/en/model_doc/sew.mdx @@ -38,8 +38,8 @@ This model was contributed by [anton-l](https://huggingface.co/anton-l). ## Documentation resources -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) ## SEWConfig diff --git a/docs/source/en/model_doc/speech_to_text_2.mdx b/docs/source/en/model_doc/speech_to_text_2.mdx index 87d7dde1be8d..745102c30bdb 100644 --- a/docs/source/en/model_doc/speech_to_text_2.mdx +++ b/docs/source/en/model_doc/speech_to_text_2.mdx @@ -96,7 +96,7 @@ See [model hub](https://huggingface.co/models?filter=speech2text2) to look for S ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) +- [Causal language modeling task guide](../tasks/language_modeling) ## Speech2Text2Config diff --git a/docs/source/en/model_doc/splinter.mdx b/docs/source/en/model_doc/splinter.mdx index 329b53ccc57b..6c89ac6fe77a 100644 --- a/docs/source/en/model_doc/splinter.mdx +++ b/docs/source/en/model_doc/splinter.mdx @@ -49,7 +49,7 @@ This model was contributed by [yuvalkirstain](https://huggingface.co/yuvalkirsta ## Documentation resources -- [Question answering task guide](./tasks/question-answering) +- [Question answering task guide](../tasks/question-answering) ## SplinterConfig diff --git a/docs/source/en/model_doc/squeezebert.mdx b/docs/source/en/model_doc/squeezebert.mdx index e793346cceee..a5f443f9a96e 100644 --- a/docs/source/en/model_doc/squeezebert.mdx +++ b/docs/source/en/model_doc/squeezebert.mdx @@ -48,11 +48,11 @@ This model was contributed by [forresti](https://huggingface.co/forresti). ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## SqueezeBertConfig diff --git a/docs/source/en/model_doc/swin.mdx b/docs/source/en/model_doc/swin.mdx index a7aa0537c76a..5f1cf1bfd086 100644 --- a/docs/source/en/model_doc/swin.mdx +++ b/docs/source/en/model_doc/swin.mdx @@ -52,7 +52,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`SwinForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) Besides that: diff --git a/docs/source/en/model_doc/swinv2.mdx b/docs/source/en/model_doc/swinv2.mdx index 703880cc084b..02ff62c68574 100644 --- a/docs/source/en/model_doc/swinv2.mdx +++ b/docs/source/en/model_doc/swinv2.mdx @@ -33,7 +33,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`Swinv2ForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) Besides that: diff --git a/docs/source/en/model_doc/switch_transformers.mdx b/docs/source/en/model_doc/switch_transformers.mdx index be156e645b49..679863d2645c 100644 --- a/docs/source/en/model_doc/switch_transformers.mdx +++ b/docs/source/en/model_doc/switch_transformers.mdx @@ -34,8 +34,8 @@ The original code can be found [here](https://github.com/google/flaxformer/tree/ ## Resources -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## SwitchTransformersConfig diff --git a/docs/source/en/model_doc/t5.mdx b/docs/source/en/model_doc/t5.mdx index 8b0bfafc00d8..f7665c11ae4a 100644 --- a/docs/source/en/model_doc/t5.mdx +++ b/docs/source/en/model_doc/t5.mdx @@ -341,7 +341,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFT5ForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/summarization) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization-tf.ipynb). - [`FlaxT5ForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/summarization). - [Summarization](https://huggingface.co/course/chapter7/5?fw=pt#summarization) chapter of the 🤗 Hugging Face course. -- [Summarization task guide](./tasks/summarization) +- [Summarization task guide](../tasks/summarization) @@ -351,7 +351,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`T5ForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/translation) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/translation.ipynb). - [`TFT5ForConditionalGeneration`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/translation) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/translation-tf.ipynb). -- [Translation task guide](./tasks/translation) +- [Translation task guide](../tasks/translation) diff --git a/docs/source/en/model_doc/tapas.mdx b/docs/source/en/model_doc/tapas.mdx index e7e660896529..fadda58957c9 100644 --- a/docs/source/en/model_doc/tapas.mdx +++ b/docs/source/en/model_doc/tapas.mdx @@ -571,8 +571,8 @@ In case of a conversational set-up, then each table-question pair must be provid ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Masked language modeling task guide](./tasks/masked_language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Masked language modeling task guide](../tasks/masked_language_modeling) ## TAPAS specific outputs [[autodoc]] models.tapas.modeling_tapas.TableQuestionAnsweringOutput diff --git a/docs/source/en/model_doc/timesformer.mdx b/docs/source/en/model_doc/timesformer.mdx index 7591d1fbc970..157f806e4263 100644 --- a/docs/source/en/model_doc/timesformer.mdx +++ b/docs/source/en/model_doc/timesformer.mdx @@ -30,7 +30,7 @@ The original code can be found [here](https://github.com/facebookresearch/TimeSf ## Documentation resources -- [Video classification task guide](./tasks/video_classification) +- [Video classification task guide](../tasks/video_classification) ## TimesformerConfig diff --git a/docs/source/en/model_doc/transfo-xl.mdx b/docs/source/en/model_doc/transfo-xl.mdx index bfd99cf1aa2b..83ce8bc76fce 100644 --- a/docs/source/en/model_doc/transfo-xl.mdx +++ b/docs/source/en/model_doc/transfo-xl.mdx @@ -60,8 +60,8 @@ TransformerXL does **not** work with *torch.nn.DataParallel* due to a bug in PyT ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Causal language modeling task guide](./tasks/language_modeling) +- [Text classification task guide](../tasks/sequence_classification) +- [Causal language modeling task guide](../tasks/language_modeling) ## TransfoXLConfig diff --git a/docs/source/en/model_doc/unispeech-sat.mdx b/docs/source/en/model_doc/unispeech-sat.mdx index 72400a0363d0..d045bcbe69d9 100644 --- a/docs/source/en/model_doc/unispeech-sat.mdx +++ b/docs/source/en/model_doc/unispeech-sat.mdx @@ -46,8 +46,8 @@ found [here](https://github.com/microsoft/UniSpeech/tree/main/UniSpeech-SAT). ## Documentation resources -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) ## UniSpeechSatConfig diff --git a/docs/source/en/model_doc/unispeech.mdx b/docs/source/en/model_doc/unispeech.mdx index 6deb274b2f16..3d170b63cefa 100644 --- a/docs/source/en/model_doc/unispeech.mdx +++ b/docs/source/en/model_doc/unispeech.mdx @@ -41,8 +41,8 @@ found [here](https://github.com/microsoft/UniSpeech/tree/main/UniSpeech). ## Documentation resources -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) ## UniSpeechConfig diff --git a/docs/source/en/model_doc/upernet.mdx b/docs/source/en/model_doc/upernet.mdx index 074189599b23..e839165e74bc 100644 --- a/docs/source/en/model_doc/upernet.mdx +++ b/docs/source/en/model_doc/upernet.mdx @@ -35,7 +35,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - Demo notebooks for UPerNet can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/UPerNet). - [`UperNetForSemanticSegmentation`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/semantic-segmentation) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/semantic_segmentation.ipynb). -- See also: [Semantic segmentation task guide](./tasks/semantic_segmentation) +- See also: [Semantic segmentation task guide](../tasks/semantic_segmentation) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/van.mdx b/docs/source/en/model_doc/van.mdx index e6f021cd9f33..e59af727589e 100644 --- a/docs/source/en/model_doc/van.mdx +++ b/docs/source/en/model_doc/van.mdx @@ -39,7 +39,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`VanForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/videomae.mdx b/docs/source/en/model_doc/videomae.mdx index efa1ed14fed2..00237055ac3d 100644 --- a/docs/source/en/model_doc/videomae.mdx +++ b/docs/source/en/model_doc/videomae.mdx @@ -43,7 +43,7 @@ review it! The resource should ideally demonstrate something new instead of dupl **Video classification** - [A notebook](https://github.com/huggingface/notebooks/blob/main/examples/video_classification.ipynb) that shows how to fine-tune a VideoMAE model on a custom dataset. -- [Video classification task guide](./tasks/video-classification) +- [Video classification task guide](../tasks/video-classification) - [A 🤗 Space](https://huggingface.co/spaces/sayakpaul/video-classification-ucf101-subset) showing how to perform inference with a video classification model. diff --git a/docs/source/en/model_doc/vit.mdx b/docs/source/en/model_doc/vit.mdx index 0244c664021d..31977a45ca95 100644 --- a/docs/source/en/model_doc/vit.mdx +++ b/docs/source/en/model_doc/vit.mdx @@ -95,7 +95,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`ViTForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). - A blog on fine-tuning [`ViTForImageClassification`] on a custom dataset can be found [here](https://huggingface.co/blog/fine-tune-vit). - More demo notebooks to fine-tune [`ViTForImageClassification`] can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/VisionTransformer). -- [Image classification task guide](./tasks/image_classification) +- [Image classification task guide](../tasks/image_classification) Besides that: diff --git a/docs/source/en/model_doc/vit_hybrid.mdx b/docs/source/en/model_doc/vit_hybrid.mdx index 07377bdb3fd4..f53b3ff10d86 100644 --- a/docs/source/en/model_doc/vit_hybrid.mdx +++ b/docs/source/en/model_doc/vit_hybrid.mdx @@ -44,7 +44,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`ViTHybridForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/vit_msn.mdx b/docs/source/en/model_doc/vit_msn.mdx index 41c2ff32d53b..f4e4afcac387 100644 --- a/docs/source/en/model_doc/vit_msn.mdx +++ b/docs/source/en/model_doc/vit_msn.mdx @@ -53,7 +53,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`ViTMSNForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). -- See also: [Image classification task guide](./tasks/image_classification) +- See also: [Image classification task guide](../tasks/image_classification) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/wav2vec2-conformer.mdx b/docs/source/en/model_doc/wav2vec2-conformer.mdx index 29f2ace678ce..cf185346197f 100644 --- a/docs/source/en/model_doc/wav2vec2-conformer.mdx +++ b/docs/source/en/model_doc/wav2vec2-conformer.mdx @@ -35,8 +35,8 @@ The original code can be found [here](https://github.com/pytorch/fairseq/tree/ma ## Documentation resources -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) ## Wav2Vec2ConformerConfig diff --git a/docs/source/en/model_doc/wav2vec2.mdx b/docs/source/en/model_doc/wav2vec2.mdx index 43210fd0aef4..837e4526e858 100644 --- a/docs/source/en/model_doc/wav2vec2.mdx +++ b/docs/source/en/model_doc/wav2vec2.mdx @@ -43,7 +43,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - A notebook on how to [leverage a pretrained Wav2Vec2 model for emotion classification](https://colab.research.google.com/github/m3hrdadfi/soxan/blob/main/notebooks/Emotion_recognition_in_Greek_speech_using_Wav2Vec2.ipynb). 🌎 - [`Wav2Vec2ForCTC`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/audio-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/audio_classification.ipynb). -- [Audio classification task guide](./tasks/audio_classification) +- [Audio classification task guide](../tasks/audio_classification) @@ -52,7 +52,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - A blog post on [finetuning XLS-R for Multi-Lingual ASR with 🤗 Transformers](https://huggingface.co/blog/fine-tune-xlsr-wav2vec2). - A notebook on how to [create YouTube captions from any video by transcribing audio with Wav2Vec2](https://colab.research.google.com/github/Muennighoff/ytclipcc/blob/main/wav2vec_youtube_captions.ipynb). 🌎 - [`Wav2Vec2ForCTC`] is supported by a notebook on [how to finetune a speech recognition model in English](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/speech_recognition.ipynb), and [how to finetune a speech recognition model in any language](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multi_lingual_speech_recognition.ipynb). -- [Automatic speech recognition task guide](./tasks/asr) +- [Automatic speech recognition task guide](../tasks/asr) 🚀 Deploy diff --git a/docs/source/en/model_doc/wavlm.mdx b/docs/source/en/model_doc/wavlm.mdx index 1157842f930c..ce58b1c965be 100644 --- a/docs/source/en/model_doc/wavlm.mdx +++ b/docs/source/en/model_doc/wavlm.mdx @@ -46,8 +46,8 @@ found [here](https://github.com/microsoft/unilm/tree/master/wavlm). ## Documentation resources -- [Audio classification task guide](./tasks/audio_classification) -- [Automatic speech recognition task guide](./tasks/asr) +- [Audio classification task guide](../tasks/audio_classification) +- [Automatic speech recognition task guide](../tasks/asr) ## WavLMConfig diff --git a/docs/source/en/model_doc/xglm.mdx b/docs/source/en/model_doc/xglm.mdx index a9883847fbd5..fb4c7a14289f 100644 --- a/docs/source/en/model_doc/xglm.mdx +++ b/docs/source/en/model_doc/xglm.mdx @@ -40,7 +40,7 @@ This model was contributed by [Suraj](https://huggingface.co/valhalla). The orig ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) +- [Causal language modeling task guide](../tasks/language_modeling) ## XGLMConfig diff --git a/docs/source/en/model_doc/xlm-prophetnet.mdx b/docs/source/en/model_doc/xlm-prophetnet.mdx index 1bfae1ec426f..ba6b91da6c43 100644 --- a/docs/source/en/model_doc/xlm-prophetnet.mdx +++ b/docs/source/en/model_doc/xlm-prophetnet.mdx @@ -54,9 +54,9 @@ Tips: ## Documentation resources -- [Causal language modeling task guide](./tasks/language_modeling) -- [Translation task guide](./tasks/translation) -- [Summarization task guide](./tasks/summarization) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Translation task guide](../tasks/translation) +- [Summarization task guide](../tasks/summarization) ## XLMProphetNetConfig diff --git a/docs/source/en/model_doc/xlm-roberta-xl.mdx b/docs/source/en/model_doc/xlm-roberta-xl.mdx index ebda86e19c2d..7c9efa593d66 100644 --- a/docs/source/en/model_doc/xlm-roberta-xl.mdx +++ b/docs/source/en/model_doc/xlm-roberta-xl.mdx @@ -30,12 +30,12 @@ This model was contributed by [Soonhwan-Kwon](https://github.com/Soonhwan-Kwon) ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## XLMRobertaXLConfig diff --git a/docs/source/en/model_doc/xlm-roberta.mdx b/docs/source/en/model_doc/xlm-roberta.mdx index 2bcff1301642..f7fe74edeb94 100644 --- a/docs/source/en/model_doc/xlm-roberta.mdx +++ b/docs/source/en/model_doc/xlm-roberta.mdx @@ -64,7 +64,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFXLMRobertaForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification-tf.ipynb). - [`FlaxXLMRobertaForSequenceClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/text-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/text_classification_flax.ipynb). - [Text classification](https://huggingface.co/docs/transformers/tasks/sequence_classification) chapter of the 🤗 Hugging Face Task Guides. -- [Text classification task guide](./tasks/sequence_classification) +- [Text classification task guide](../tasks/sequence_classification) @@ -72,13 +72,13 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFXLMRobertaForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/token-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb). - [`FlaxXLMRobertaForTokenClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/token-classification). - [Token classification](https://huggingface.co/course/chapter7/2?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Token classification task guide](./tasks/token_classification) +- [Token classification task guide](../tasks/token_classification) - [`XLMRobertaForCausalLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb). - [Causal language modeling](https://huggingface.co/docs/transformers/tasks/language_modeling) chapter of the 🤗 Hugging Face Task Guides. -- [Causal language modeling task guide](./tasks/language_modeling) +- [Causal language modeling task guide](../tasks/language_modeling) @@ -86,7 +86,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFXLMRobertaForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling#run_mlmpy) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb). - [`FlaxXLMRobertaForMaskedLM`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling#masked-language-modeling) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/masked_language_modeling_flax.ipynb). - [Masked language modeling](https://huggingface.co/course/chapter7/3?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Masked language modeling](./tasks/masked_language_modeling) +- [Masked language modeling](../tasks/masked_language_modeling) @@ -94,13 +94,13 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - [`TFXLMRobertaForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/question-answering) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb). - [`FlaxXLMRobertaForQuestionAnswering`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/flax/question-answering). - [Question answering](https://huggingface.co/course/chapter7/7?fw=pt) chapter of the 🤗 Hugging Face Course. -- [Question answering task guide](./tasks/question_answering) +- [Question answering task guide](../tasks/question_answering) **Multiple choice** - [`XLMRobertaForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice.ipynb). - [`TFXLMRobertaForMultipleChoice`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/multiple-choice) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/multiple_choice-tf.ipynb). -- [Multiple choice task guide](./tasks/multiple_choice) +- [Multiple choice task guide](../tasks/multiple_choice) 🚀 Deploy diff --git a/docs/source/en/model_doc/xlm.mdx b/docs/source/en/model_doc/xlm.mdx index 04923d2285e3..846ca73b6b9e 100644 --- a/docs/source/en/model_doc/xlm.mdx +++ b/docs/source/en/model_doc/xlm.mdx @@ -57,12 +57,12 @@ This model was contributed by [thomwolf](https://huggingface.co/thomwolf). The o ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## XLMConfig diff --git a/docs/source/en/model_doc/xlnet.mdx b/docs/source/en/model_doc/xlnet.mdx index 2bebc4719b56..599c865c71dd 100644 --- a/docs/source/en/model_doc/xlnet.mdx +++ b/docs/source/en/model_doc/xlnet.mdx @@ -56,11 +56,11 @@ This model was contributed by [thomwolf](https://huggingface.co/thomwolf). The o ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## XLNetConfig diff --git a/docs/source/en/model_doc/xmod.mdx b/docs/source/en/model_doc/xmod.mdx index 7c6573469298..c240889b025a 100644 --- a/docs/source/en/model_doc/xmod.mdx +++ b/docs/source/en/model_doc/xmod.mdx @@ -80,12 +80,12 @@ model.set_default_language("de_DE") ## Resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Causal language modeling task guide](./tasks/language_modeling) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Causal language modeling task guide](../tasks/language_modeling) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## XmodConfig diff --git a/docs/source/en/model_doc/yolos.mdx b/docs/source/en/model_doc/yolos.mdx index bab338860e94..ec5cf6cbb492 100644 --- a/docs/source/en/model_doc/yolos.mdx +++ b/docs/source/en/model_doc/yolos.mdx @@ -39,7 +39,7 @@ A list of official Hugging Face and community (indicated by 🌎) resources to h - All example notebooks illustrating inference + fine-tuning [`YolosForObjectDetection`] on a custom dataset can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/YOLOS). -- See also: [Object detection task guide](./tasks/object_detection) +- See also: [Object detection task guide](../tasks/object_detection) If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. diff --git a/docs/source/en/model_doc/yoso.mdx b/docs/source/en/model_doc/yoso.mdx index 575c1b49165e..66b05b33cb4d 100644 --- a/docs/source/en/model_doc/yoso.mdx +++ b/docs/source/en/model_doc/yoso.mdx @@ -52,11 +52,11 @@ This model was contributed by [novice03](https://huggingface.co/novice03). The o ## Documentation resources -- [Text classification task guide](./tasks/sequence_classification) -- [Token classification task guide](./tasks/token_classification) -- [Question answering task guide](./tasks/question_answering) -- [Masked language modeling task guide](./tasks/masked_language_modeling) -- [Multiple choice task guide](./tasks/multiple_choice) +- [Text classification task guide](../tasks/sequence_classification) +- [Token classification task guide](../tasks/token_classification) +- [Question answering task guide](../tasks/question_answering) +- [Masked language modeling task guide](../tasks/masked_language_modeling) +- [Multiple choice task guide](../tasks/multiple_choice) ## YosoConfig From 52e69dde37d4ad01749766a299aeb5efa3f1f8a7 Mon Sep 17 00:00:00 2001 From: Ali Hassani <68103095+alihassanijr@users.noreply.github.com> Date: Fri, 17 Mar 2023 11:07:55 -0400 Subject: [PATCH 128/277] Fix natten (#22229) * Add kernel size to NATTEN's QK arguments. The new NATTEN 0.14.5 supports PyTorch 2.0, but also adds an additional argument to the QK operation to allow optional RPBs. This ends up failing NATTEN tests. This commit adds NATTEN back to circleci and adds the arguments to get it working again. * Force NATTEN >= 0.14.5 --- .circleci/create_circleci_config.py | 3 +-- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- src/transformers/models/dinat/modeling_dinat.py | 2 +- src/transformers/models/nat/modeling_nat.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 6a45f94a6578..0b26762b080a 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -374,8 +374,7 @@ def job_name(self): "pip install 'git+https://github.com/facebookresearch/detectron2.git'", "sudo apt install tesseract-ocr", "pip install pytesseract", - # wait until natten is ready for torch 2.0.0 - # "pip install natten", + "pip install natten", ], tests_to_run=[ "tests/models/*layoutlmv*", diff --git a/setup.py b/setup.py index 943bb196b5d6..c28387a3d454 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,7 @@ "keras-nlp>=0.3.1", "librosa", "nltk", - "natten>=0.14.4", + "natten>=0.14.5", "numpy>=1.17", "onnxconverter-common", "onnxruntime-tools>=1.4.2", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 79f9118ae847..aa638a6a9f60 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -35,7 +35,7 @@ "keras-nlp": "keras-nlp>=0.3.1", "librosa": "librosa", "nltk": "nltk", - "natten": "natten>=0.14.4", + "natten": "natten>=0.14.5", "numpy": "numpy>=1.17", "onnxconverter-common": "onnxconverter-common", "onnxruntime-tools": "onnxruntime-tools>=1.4.2", diff --git a/src/transformers/models/dinat/modeling_dinat.py b/src/transformers/models/dinat/modeling_dinat.py index 95191d52b5f6..efeb68846fce 100644 --- a/src/transformers/models/dinat/modeling_dinat.py +++ b/src/transformers/models/dinat/modeling_dinat.py @@ -347,7 +347,7 @@ def forward( query_layer = query_layer / math.sqrt(self.attention_head_size) # Compute NA between "query" and "key" to get the raw attention scores, and add relative positional biases. - attention_scores = natten2dqkrpb(query_layer, key_layer, self.rpb, self.dilation) + attention_scores = natten2dqkrpb(query_layer, key_layer, self.rpb, self.kernel_size, self.dilation) # Normalize the attention scores to probabilities. attention_probs = nn.functional.softmax(attention_scores, dim=-1) diff --git a/src/transformers/models/nat/modeling_nat.py b/src/transformers/models/nat/modeling_nat.py index 4b34fe730c16..3a93b81e4bc5 100644 --- a/src/transformers/models/nat/modeling_nat.py +++ b/src/transformers/models/nat/modeling_nat.py @@ -339,7 +339,7 @@ def forward( query_layer = query_layer / math.sqrt(self.attention_head_size) # Compute NA between "query" and "key" to get the raw attention scores, and add relative positional biases. - attention_scores = natten2dqkrpb(query_layer, key_layer, self.rpb, 1) + attention_scores = natten2dqkrpb(query_layer, key_layer, self.rpb, self.kernel_size, 1) # Normalize the attention scores to probabilities. attention_probs = nn.functional.softmax(attention_scores, dim=-1) From 9607a01638ec4448878c3a3ac05f0c2774bad1d7 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 17 Mar 2023 16:54:27 +0100 Subject: [PATCH 129/277] Revert "Use `dash==2.8.1` for now for daily CI" (#22233) Revert "Use `dash==2.8.1` for now for daily CI (#22227)" This reverts commit 53218671d968235ff320a4b03f7753972a637299. --- docker/transformers-all-latest-gpu/Dockerfile | 3 --- docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile | 3 --- docker/transformers-pytorch-gpu/Dockerfile | 3 --- 3 files changed, 9 deletions(-) diff --git a/docker/transformers-all-latest-gpu/Dockerfile b/docker/transformers-all-latest-gpu/Dockerfile index da2ed78bd080..9f30f4531b0a 100644 --- a/docker/transformers-all-latest-gpu/Dockerfile +++ b/docker/transformers-all-latest-gpu/Dockerfile @@ -59,9 +59,6 @@ RUN python3 -m pip install --no-cache-dir decord av==9.2.0 ## For `dinat` model #RUN python3 -m pip install --no-cache-dir natten -f https://shi-labs.com/natten/wheels/$CUDA/ -# dash 2.9.0 has some issue. -RUN python3 -m pip install -U dash==2.8.1 - # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile index 09160b96aad6..2fa1317f8275 100644 --- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile +++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile @@ -40,9 +40,6 @@ RUN python3 -m pip uninstall -y deepspeed # TODO: Find out why test fail. RUN DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1 -# dash 2.9.0 has some issue. -RUN python3 -m pip install -U dash==2.8.1 - # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop diff --git a/docker/transformers-pytorch-gpu/Dockerfile b/docker/transformers-pytorch-gpu/Dockerfile index a5115890a763..24564aca6fae 100644 --- a/docker/transformers-pytorch-gpu/Dockerfile +++ b/docker/transformers-pytorch-gpu/Dockerfile @@ -29,9 +29,6 @@ RUN python3 -m pip uninstall -y tensorflow flax RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract RUN python3 -m pip install -U "itsdangerous<2.1.0" -# dash 2.9.0 has some issue. -RUN python3 -m pip install -U dash==2.8.1 - # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop From 59090ff2c2b23f0bb05edef5057e795bf7f9ebd5 Mon Sep 17 00:00:00 2001 From: Guangyuan Ma <63697972+ma787639046@users.noreply.github.com> Date: Sat, 18 Mar 2023 01:56:32 +0800 Subject: [PATCH 130/277] Fix Unnecessary move of tensors from CPU to GPU in LlamaRotaryEmbedding (#22234) push --- src/transformers/models/llama/modeling_llama.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index 5cdea1b787f6..69d8e86693b6 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -99,8 +99,8 @@ def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): freqs = torch.einsum("i,j->ij", t, self.inv_freq) # Different from paper, but it uses a different permutation in order to obtain the same calculation emb = torch.cat((freqs, freqs), dim=-1) - self.cos_cached = emb.cos()[None, None, :, :] - self.sin_cached = emb.sin()[None, None, :, :] + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) def forward(self, x, seq_len=None): # x: [bs, num_attention_heads, seq_len, head_size] @@ -111,11 +111,11 @@ def forward(self, x, seq_len=None): freqs = torch.einsum("i,j->ij", t, self.inv_freq) # Different from paper, but it uses a different permutation in order to obtain the same calculation emb = torch.cat((freqs, freqs), dim=-1).to(x.device) - self.cos_cached = emb.cos()[None, None, :, :].to(dtype=x.dtype) - self.sin_cached = emb.sin()[None, None, :, :].to(dtype=x.dtype) + self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False) + self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False) return ( - self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype, device=x.device), - self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype, device=x.device), + self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype), + self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype), ) From 0a9926d7fd918c8814fc908aebf438f15a7872c0 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 17 Mar 2023 11:02:55 -0700 Subject: [PATCH 131/277] [trainer] param count for deepspeed zero3 (#22193) [trainer] param count for zero3 --- src/transformers/trainer.py | 5 ++--- src/transformers/trainer_pt_utils.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index efd8d0fc3a4e..9549822d92fc 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -97,6 +97,7 @@ distributed_broadcast_scalars, distributed_concat, find_batch_size, + get_model_param_count, get_module_class_from_name, get_parameter_names, nested_concat, @@ -1744,9 +1745,7 @@ def _inner_training_loop( logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_train_batch_size}") logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") logger.info(f" Total optimization steps = {max_steps}") - logger.info( - f" Number of trainable parameters = {sum(p.numel() for p in model.parameters() if p.requires_grad)}" - ) + logger.info(f" Number of trainable parameters = {get_model_param_count(model, trainable_only=True)}") self.state.epoch = 0 start_time = time.time() diff --git a/src/transformers/trainer_pt_utils.py b/src/transformers/trainer_pt_utils.py index eefbb5268345..dee1dce0f6f7 100644 --- a/src/transformers/trainer_pt_utils.py +++ b/src/transformers/trainer_pt_utils.py @@ -35,6 +35,7 @@ from torch.utils.data import Dataset, IterableDataset, RandomSampler, Sampler from torch.utils.data.distributed import DistributedSampler +from .deepspeed import is_deepspeed_zero3_enabled from .tokenization_utils_base import BatchEncoding from .utils import is_sagemaker_mp_enabled, is_torch_tpu_available, is_training_run_on_sagemaker, logging @@ -1032,6 +1033,23 @@ def save_state(self): self.state.save_to_json(path) +def get_model_param_count(model, trainable_only=False): + """ + Calculate model's total param count. If trainable_only is True then count only those requiring grads + """ + if is_deepspeed_zero3_enabled(): + + def numel(p): + return p.ds_numel + + else: + + def numel(p): + return p.numel() + + return sum(numel(p) for p in model.parameters() if not trainable_only or p.requires_grad) + + def get_parameter_names(model, forbidden_layer_types): """ Returns the names of the model parameters that are not inside a forbidden layer. From 12afa8b663fc815b3af2433a7125006b580e1fee Mon Sep 17 00:00:00 2001 From: Pasquale Minervini Date: Mon, 20 Mar 2023 13:00:05 +0100 Subject: [PATCH 132/277] Update training_args.py -- a nightly install is not required anymore for torch.compile (#22266) Update training_args.py A nightly install is not required anymore for `torch.compile`. --- src/transformers/training_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index ce65706c0c5f..ea91ddfe8137 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -557,7 +557,7 @@ class TrainingArguments: Whether to use Apple Silicon chip based `mps` device. torch_compile (`bool`, *optional*, defaults to `False`): Whether or not to compile the model using PyTorch 2.0 - [`torch.compile`](https://pytorch.org/get-started/pytorch-2.0/) (requires a nighlty install of PyTorch). + [`torch.compile`](https://pytorch.org/get-started/pytorch-2.0/). This will use the best defaults for the [`torch.compile` API](https://pytorch.org/docs/2.0/generated/torch.compile.html?highlight=torch+compile#torch.compile). You From bf2fc8003c3e7e039585ba000d542f5e12605b2d Mon Sep 17 00:00:00 2001 From: yesinkim Date: Mon, 20 Mar 2023 21:17:31 +0900 Subject: [PATCH 133/277] [Docs] fix typos in some tokenizer docs (#22256) [Docs] fix typos Co-authored-by: yesinkim --- src/transformers/models/longformer/tokenization_longformer.py | 2 +- .../models/longformer/tokenization_longformer_fast.py | 2 +- src/transformers/models/roberta/tokenization_roberta.py | 2 +- src/transformers/models/roberta/tokenization_roberta_fast.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/longformer/tokenization_longformer.py b/src/transformers/models/longformer/tokenization_longformer.py index 5ff6f70afdea..69bc50595387 100644 --- a/src/transformers/models/longformer/tokenization_longformer.py +++ b/src/transformers/models/longformer/tokenization_longformer.py @@ -124,7 +124,7 @@ class LongformerTokenizer(PreTrainedTokenizer): >>> from transformers import LongformerTokenizer >>> tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096") >>> tokenizer("Hello world")['input_ids'] - [0, 31414, 232, 328, 2] + [0, 31414, 232, 2] >>> tokenizer(" Hello world")['input_ids'] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/longformer/tokenization_longformer_fast.py b/src/transformers/models/longformer/tokenization_longformer_fast.py index 5d20caf8c2df..dfe1b08e1458 100644 --- a/src/transformers/models/longformer/tokenization_longformer_fast.py +++ b/src/transformers/models/longformer/tokenization_longformer_fast.py @@ -100,7 +100,7 @@ class LongformerTokenizerFast(PreTrainedTokenizerFast): >>> from transformers import LongformerTokenizerFast >>> tokenizer = LongformerTokenizerFast.from_pretrained("allenai/longformer-base-4096") >>> tokenizer("Hello world")['input_ids'] - [0, 31414, 232, 328, 2] + [0, 31414, 232, 2] >>> tokenizer(" Hello world")['input_ids'] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/roberta/tokenization_roberta.py b/src/transformers/models/roberta/tokenization_roberta.py index d291a2f9d97a..e8d4a751bc19 100644 --- a/src/transformers/models/roberta/tokenization_roberta.py +++ b/src/transformers/models/roberta/tokenization_roberta.py @@ -115,7 +115,7 @@ class RobertaTokenizer(PreTrainedTokenizer): >>> from transformers import RobertaTokenizer >>> tokenizer = RobertaTokenizer.from_pretrained("roberta-base") >>> tokenizer("Hello world")['input_ids'] - [0, 31414, 232, 328, 2] + [0, 31414, 232, 2] >>> tokenizer(" Hello world")['input_ids'] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/roberta/tokenization_roberta_fast.py b/src/transformers/models/roberta/tokenization_roberta_fast.py index 49311b3aeff9..32cc66d750b0 100644 --- a/src/transformers/models/roberta/tokenization_roberta_fast.py +++ b/src/transformers/models/roberta/tokenization_roberta_fast.py @@ -85,7 +85,7 @@ class RobertaTokenizerFast(PreTrainedTokenizerFast): >>> from transformers import RobertaTokenizerFast >>> tokenizer = RobertaTokenizerFast.from_pretrained("roberta-base") >>> tokenizer("Hello world")['input_ids'] - [0, 31414, 232, 328, 2] + [0, 31414, 232, 2] >>> tokenizer(" Hello world")['input_ids'] [0, 20920, 232, 2] ``` From 177fd5032e1a757db321bf5b5570968c824c6a39 Mon Sep 17 00:00:00 2001 From: Nicola Procopio Date: Mon, 20 Mar 2023 14:16:07 +0100 Subject: [PATCH 134/277] Italian translation perf_infer_cpu (#22243) * added translated files added perf_train_cpu and perf_train_cpu_many * updated toctree * updated toctree * added file perf_infer_cpu.medx * italian translation perf_infer_cpu.mdx --- docs/source/it/_toctree.yml | 2 + docs/source/it/perf_infer_cpu.mdx | 75 +++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 docs/source/it/perf_infer_cpu.mdx diff --git a/docs/source/it/_toctree.yml b/docs/source/it/_toctree.yml index abb252b321b7..e0920248fffa 100644 --- a/docs/source/it/_toctree.yml +++ b/docs/source/it/_toctree.yml @@ -37,6 +37,8 @@ title: Addestramento efficiente su CPU - local: perf_train_cpu_many title: Addestramento efficiente su multiple CPU + - local: perf_infer_cpu + title: Inferenza Efficiente su CPU - local: big_models title: Istanziare un big model - local: migration diff --git a/docs/source/it/perf_infer_cpu.mdx b/docs/source/it/perf_infer_cpu.mdx new file mode 100644 index 000000000000..1423b8f0552c --- /dev/null +++ b/docs/source/it/perf_infer_cpu.mdx @@ -0,0 +1,75 @@ + + +# Inferenza Efficiente su CPU + +Questa guida si concentra sull'inferenza di modelli di grandi dimensioni in modo efficiente sulla CPU. + +## `BetterTransformer` per inferenza più rapida + +Abbiamo integrato di recente `BetterTransformer` per fare inferenza più rapidamente con modelli per testi, immagini e audio. Visualizza la documentazione sull'integrazione [qui](https://huggingface.co/docs/optimum/bettertransformer/overview) per maggiori dettagli. + +## PyTorch JIT-mode (TorchScript) + +TorchScript è un modo di creare modelli serializzabili e ottimizzabili da codice PyTorch. Ogni programmma TorchScript può esere salvato da un processo Python e caricato in un processo dove non ci sono dipendenze Python. +Comparandolo con l'eager mode di default, jit mode in PyTorch normalmente fornisce prestazioni migliori per l'inferenza del modello da parte di metodologie di ottimizzazione come la operator fusion. + +Per una prima introduzione a TorchScript, vedi la Introduction to [PyTorch TorchScript tutorial](https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html#tracing-modules). + +### IPEX Graph Optimization con JIT-mode + +Intel® Extension per PyTorch fornnisce ulteriori ottimizzazioni in jit mode per i modelli della serie Transformers. Consigliamo vivamente agli utenti di usufruire dei vantaggi di Intel® Extension per PyTorch con jit mode. Alcuni operator patterns usati fequentemente dai modelli Transformers models sono già supportati in Intel® Extension per PyTorch con jit mode fusions. Questi fusion patterns come Multi-head-attention fusion, Concat Linear, Linear+Add, Linear+Gelu, Add+LayerNorm fusion and etc. sono abilitati e hanno buone performance. I benefici della fusion è fornito agli utenti in modo trasparente. In base alle analisi, il ~70% dei problemi più popolari in NLP question-answering, text-classification, and token-classification possono avere benefici sulle performance grazie ai fusion patterns sia per Float32 precision che per BFloat16 Mixed precision. + +Vedi maggiori informazioni per [IPEX Graph Optimization](https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/features/graph_optimization.html). + +#### Installazione di IPEX + +I rilasci di IPEX seguono PyTorch, verifica i vari approcci per [IPEX installation](https://intel.github.io/intel-extension-for-pytorch/). + +### Utilizzo del JIT-mode + +Per abilitare JIT-mode in Trainer per evaluation e prediction, devi aggiungere `jit_mode_eval` negli argomenti di Trainer. + + + +per PyTorch >= 1.14.0. JIT-mode potrebe giovare a qualsiasi modello di prediction e evaluaion visto che il dict input è supportato in jit.trace + +per PyTorch < 1.14.0. JIT-mode potrebbe giovare ai modelli il cui ordine dei parametri corrisponde all'ordine delle tuple in ingresso in jit.trace, come i modelli per question-answering. +Nel caso in cui l'ordine dei parametri seguenti non corrisponda all'ordine delle tuple in ingresso in jit.trace, come nei modelli di text-classification, jit.trace fallirà e lo cattureremo con una eccezione al fine di renderlo un fallback. Il logging è usato per notificare gli utenti. + + + +Trovi un esempo con caso d'uso in [Transformers question-answering](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering) + +- Inference using jit mode on CPU: + +
python run_qa.py \
+--model_name_or_path csarron/bert-base-uncased-squad-v1 \
+--dataset_name squad \
+--do_eval \
+--max_seq_length 384 \
+--doc_stride 128 \
+--output_dir /tmp/ \
+--no_cuda \
+--jit_mode_eval
+ +- Inference with IPEX using jit mode on CPU: + +
python run_qa.py \
+--model_name_or_path csarron/bert-base-uncased-squad-v1 \
+--dataset_name squad \
+--do_eval \
+--max_seq_length 384 \
+--doc_stride 128 \
+--output_dir /tmp/ \
+--no_cuda \
+--use_ipex \
+--jit_mode_eval
From 1c588fe6dd9eb72924c43e527f4e84fde3844afe Mon Sep 17 00:00:00 2001 From: heya5 Date: Mon, 20 Mar 2023 21:17:34 +0800 Subject: [PATCH 135/277] [Trainer] Add optional communication backends for torch.distributed when using GPU (#22247) Update training_args.py --- src/transformers/training_args.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index ea91ddfe8137..4c13ad0af415 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -1641,7 +1641,10 @@ def _setup_devices(self) -> "torch.device": # Here, we'll use torch.distributed. # Initializes the distributed backend which will take care of synchronizing nodes/GPUs if not torch.distributed.is_initialized(): - torch.distributed.init_process_group(backend="nccl", timeout=self.ddp_timeout_delta) + if self.xpu_backend and self.xpu_backend in ("mpi", "gloo"): + torch.distributed.init_process_group(backend=self.xpu_backend, timeout=self.ddp_timeout_delta) + else: + torch.distributed.init_process_group(backend="nccl", timeout=self.ddp_timeout_delta) device = torch.device("cuda", self.local_rank) self._n_gpu = 1 From 484c9934b11e0acd04baf1220f7ecbd23c09e0e5 Mon Sep 17 00:00:00 2001 From: yqy2001 <55196500+yqy2001@users.noreply.github.com> Date: Mon, 20 Mar 2023 22:26:50 +0800 Subject: [PATCH 136/277] Fix the gradient checkpointing bug of the llama model (#22270) fix grad ckpt bug of llama --- src/transformers/models/llama/modeling_llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index 69d8e86693b6..d46ae1ee09f3 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -372,7 +372,7 @@ def _init_weights(self, module): module.weight.data[module.padding_idx].zero_() def _set_gradient_checkpointing(self, module, value=False): - if isinstance(module, (LlamaDecoderLayer)): + if isinstance(module, LlamaModel): module.gradient_checkpointing = value From ef5f2a50a79ef9a464fbf9cad31d9a594a2b4b19 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:24:17 -0400 Subject: [PATCH 137/277] Fix balanced and auto device_map (#22271) --- src/transformers/modeling_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 697b3deb14d7..901677a604dd 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2563,7 +2563,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P elif device_map in ["balanced", "balanced_low_0"] and get_balanced_memory is None: raise ValueError(f"`device_map={device_map}` requires a source install of Accelerate.") - kwargs = {"no_split_module_classes": no_split_modules, "max_memory": max_memory} + kwargs = {"no_split_module_classes": no_split_modules} if "special_dtypes" in inspect.signature(infer_auto_device_map).parameters: kwargs["special_dtypes"] = special_dtypes elif len(special_dtypes) > 0: @@ -2578,6 +2578,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P low_zero=(device_map == "balanced_low_0"), **kwargs, ) + kwargs["max_memory"] = max_memory # Make sure tied weights are tied before creating the device map. model.tie_weights() device_map = infer_auto_device_map(model, dtype=torch_dtype if not load_in_8bit else torch.int8, **kwargs) From b2543ccdcf7421d6623e499d33691e15810d5406 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:30:36 -0400 Subject: [PATCH 138/277] Rework a bit the LLaMA conversion script (#22236) * Update LLaMA conversion script * Doc * Fix the weight size for the 13B checkpoint * Update src/transformers/models/llama/convert_llama_weights_to_hf.py Co-authored-by: Lysandre Debut --------- Co-authored-by: Lysandre Debut --- docs/source/en/model_doc/llama.mdx | 7 +- .../llama/convert_llama_weights_to_hf.py | 111 +++++++++--------- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/docs/source/en/model_doc/llama.mdx b/docs/source/en/model_doc/llama.mdx index 92c03be29f62..158ee2b1aa3d 100644 --- a/docs/source/en/model_doc/llama.mdx +++ b/docs/source/en/model_doc/llama.mdx @@ -35,10 +35,13 @@ python src/transformers/models/llama/convert_llama_weights_to_hf.py \ ```python from transformers import LlamaForCausalLM, LlamaTokenizer -tokenizer = LlamaTokenizer.from_pretrained("/output/path/tokenizer/") -model = LlamaForCausalLM.from_pretrained("/output/path/llama-7b/") +tokenizer = LlamaTokenizer.from_pretrained("/output/path") +model = LlamaForCausalLM.from_pretrained("/output/path") ``` +Note that executing the script requires enough CPU RAM to host the whole model in float16 precision (even if the biggest versions +come in several checkpoints they each contain a part of each weight of the model, so we need to load them all in RAM). For the 65B model, it's thus 130GB of RAM needed. + - The LLaMA tokenizer is based on [sentencepiece](https://github.com/google/sentencepiece). One quirk of sentencepiece is that when decoding a sequence, if the first token is the start of the word (e.g. "Banana"), the tokenizer does not prepend the prefix space to the string. To have the tokenizer output the prefix space, set `decode_with_prefix_space=True` in the `LlamaTokenizer` object or in the tokenizer configuration. This model was contributed by [zphang](https://huggingface.co/zphang) with contributions from [BlackSamorez](https://huggingface.co/BlackSamorez). The code of the implementation in Hugging Face is based on GPT-NeoX [here](https://github.com/EleutherAI/gpt-neox). The original code of the authors can be found [here](https://github.com/facebookresearch/llama). diff --git a/src/transformers/models/llama/convert_llama_weights_to_hf.py b/src/transformers/models/llama/convert_llama_weights_to_hf.py index 521ea3dbb660..b5612777936b 100644 --- a/src/transformers/models/llama/convert_llama_weights_to_hf.py +++ b/src/transformers/models/llama/convert_llama_weights_to_hf.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import argparse +import gc import json import math import os @@ -19,22 +20,28 @@ import torch +from transformers import LlamaConfig, LlamaForCausalLM + """ Sample usage: - ``` - python src/transformers/models/llama/convert_llama_weights_to_hf.py \ - --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path - ``` +``` +python src/transformers/models/llama/convert_llama_weights_to_hf.py \ + --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path +``` Thereafter, models can be loaded via: - ``` - tokenizer = transformers.LlamaTokenizer.from_pretrained("/output/path/tokenizer/") +```py +from transformers import LlamaForCausalLM, LlamaForTokenizer + +model = LlamaForCausalLM.from_pretrained("/output/path") +tokenizer = LlamaTokenizer.from_pretrained("/output/path") +``` - model = transformers.LlamaForCausalLM.from_pretrained("/output/path/llama-7b/") - ``` +Important note: you need to be able to host the whole model in RAM to execute this script (even if the biggest versions +come in several checkpoints they each contain a part of each weight of the model, so we need to load them all in RAM). """ INTERMEDIATE_SIZE_MAP = { @@ -66,8 +73,9 @@ def write_json(text, path): def write_model(model_path, input_base_path, model_size): - assert model_size in NUM_SHARDS os.makedirs(model_path, exist_ok=True) + tmp_model_path = os.path.join(model_path, "tmp") + os.makedirs(tmp_model_path, exist_ok=True) params = read_json(os.path.join(input_base_path, "params.json")) num_shards = NUM_SHARDS[model_size] @@ -83,6 +91,7 @@ def write_model(model_path, input_base_path, model_size): def permute(w): return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim) + print(f"Fetching all parameters from the checkpoint at {input_base_path}.") # Load weights if model_size == "7B": # Not shared @@ -97,10 +106,7 @@ def permute(w): param_count = 0 index_dict = {"weight_map": {}} for layer_i in range(n_layers): - filename = "pytorch_model-{:05d}-of-{:05d}.bin".format( - layer_i + 1, - n_layers + 1, - ) + filename = f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin" if model_size == "7B": # Unsharded state_dict = { @@ -120,11 +126,15 @@ def permute(w): } else: # Sharded + # Note that in the 13B checkpoint, not cloning the two following weights will result in the checkpoint + # becoming 37GB instead of 26GB for some reason. state_dict = { - f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][f"layers.{layer_i}.attention_norm.weight"], + f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][ + f"layers.{layer_i}.attention_norm.weight" + ].clone(), f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][ f"layers.{layer_i}.ffn_norm.weight" - ], + ].clone(), } state_dict[f"model.layers.{layer_i}.self_attn.q_proj.weight"] = permute( torch.cat( @@ -169,12 +179,9 @@ def permute(w): for k, v in state_dict.items(): index_dict["weight_map"][k] = filename param_count += v.numel() - torch.save(state_dict, os.path.join(model_path, filename)) + torch.save(state_dict, os.path.join(tmp_model_path, filename)) - filename = "pytorch_model-{:05d}-of-{:05d}.bin".format( - n_layers + 1, - n_layers + 1, - ) + filename = f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin" if model_size == "7B": # Unsharded state_dict = { @@ -194,48 +201,38 @@ def permute(w): for k, v in state_dict.items(): index_dict["weight_map"][k] = filename param_count += v.numel() - torch.save(state_dict, os.path.join(model_path, filename)) + torch.save(state_dict, os.path.join(tmp_model_path, filename)) # Write configs index_dict["metadata"] = {"total_size": param_count * 2} - write_json(index_dict, os.path.join(model_path, "pytorch_model.bin.index.json")) - config_out = { - "architectures": ["LlamaForCausalLM"], - "bos_token_id": 1, - "eos_token_id": 2, - "hidden_act": "silu", - "hidden_size": dim, - "intermediate_size": compute_intermediate_size(dim), - "initializer_range": 0.02, - "max_sequence_length": 2048, - "model_type": "llama", - "num_attention_heads": params["n_heads"], - "num_hidden_layers": params["n_layers"], - "pad_token_id": 0, - "rms_norm_eps": params["norm_eps"], - "torch_dtype": "float16", - "transformers_version": "4.27.0.dev0", - "use_cache": True, - "vocab_size": 32000, - } - write_json( - config_out, - os.path.join(model_path, "config.json"), - ) - generation_config = { - "_from_model_config": True, - "bos_token_id": 1, - "eos_token_id": 2, - "pad_token_id": 0, - "transformers_version": "4.27.0.dev0", - } - write_json( - generation_config, - os.path.join(model_path, "generation_config.json"), + write_json(index_dict, os.path.join(tmp_model_path, "pytorch_model.bin.index.json")) + + config = LlamaConfig( + hidden_size=dim, + intermediate_size=compute_intermediate_size(dim), + num_attention_heads=params["n_heads"], + num_hidden_layers=params["n_layers"], + rms_norm_eps=params["norm_eps"], ) + config.save_pretrained(tmp_model_path) + + # Make space so we can load the model properly now. + del state_dict + del loaded + gc.collect() + + print("Loading the checkpoint in a Llama model.") + model = LlamaForCausalLM.from_pretrained(tmp_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) + # Avoid saving this as part of the config. + del model.config._name_or_path + + print("Saving in the Transformers format.") + model.save_pretrained(model_path) + shutil.rmtree(tmp_model_path) def write_tokenizer(tokenizer_path, input_tokenizer_path): + print(f"Fetching the tokenizer from {input_tokenizer_path}.") os.makedirs(tokenizer_path, exist_ok=True) write_json({}, os.path.join(tokenizer_path, "special_tokens_map.json")) write_json( @@ -268,12 +265,12 @@ def main(): args = parser.parse_args() if args.model_size != "tokenizer_only": write_model( - model_path=os.path.join(args.output_dir, "llama-{}".format(args.model_size).lower()), + model_path=args.output_dir, input_base_path=os.path.join(args.input_dir, args.model_size), model_size=args.model_size, ) write_tokenizer( - tokenizer_path=os.path.join(args.output_dir, "tokenizer"), + tokenizer_path=args.output_dir, input_tokenizer_path=os.path.join(args.input_dir, "tokenizer.model"), ) From 71b06fa7071d5e682f8aefd7d8ee43ce14c22960 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:30:46 -0400 Subject: [PATCH 139/277] Proper map location for optimizer load (#22273) * Proper map location for optimizer load * What happened to my code? --- src/transformers/trainer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 9549822d92fc..c49e004f6e59 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -2433,8 +2433,12 @@ def opt_load_hook(mod, opt): self.model_wrapped.register_post_step_hook(opt_load_hook) else: + # We use the CPU when training on one GPU to avoid OOM for GPU RAM when training big models. + # In distributed training however, we load directly on each GPU and risk the GPU OOM as it's more + # likely to get OOM on CPU (since we load num_gpu times the optimizer state + map_location = self.args.device if self.args.world_size > 1 else "cpu" self.optimizer.load_state_dict( - torch.load(os.path.join(checkpoint, OPTIMIZER_NAME), map_location="cpu") + torch.load(os.path.join(checkpoint, OPTIMIZER_NAME), map_location=map_location) ) with warnings.catch_warnings(record=True) as caught_warnings: self.lr_scheduler.load_state_dict(torch.load(os.path.join(checkpoint, SCHEDULER_NAME))) From 05fa5c7fdf59d6615a923c9f92ff4922a0a8f78f Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Mon, 20 Mar 2023 17:07:31 +0000 Subject: [PATCH 140/277] Fix doc links (#22274) --- docs/source/en/tasks/image_classification.mdx | 6 +++--- docs/source/en/tasks/multiple_choice.mdx | 2 +- docs/source/en/tasks/semantic_segmentation.mdx | 4 ++-- docs/source/en/tasks/sequence_classification.mdx | 6 +++--- docs/source/en/tasks/summarization.mdx | 10 +++++----- docs/source/en/tasks/token_classification.mdx | 4 ++-- docs/source/en/tasks/translation.mdx | 8 ++++---- docs/source/es/tasks/summarization.mdx | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/source/en/tasks/image_classification.mdx b/docs/source/en/tasks/image_classification.mdx index c3faa6145e16..fbc8ab678798 100644 --- a/docs/source/en/tasks/image_classification.mdx +++ b/docs/source/en/tasks/image_classification.mdx @@ -403,9 +403,9 @@ Configure the model for training with `compile()`: >>> model.compile(optimizer=optimizer, loss=loss) ``` -To compute the accuracy from the predictions and push your model to the 🤗 Hub, use [Keras callbacks](./main_classes/keras_callbacks). -Pass your `compute_metrics` function to [KerasMetricCallback](./main_classes/keras_callbacks#transformers.KerasMetricCallback), -and use the [PushToHubCallback](./main_classes/keras_callbacks#transformers.PushToHubCallback) to upload the model: +To compute the accuracy from the predictions and push your model to the 🤗 Hub, use [Keras callbacks](../main_classes/keras_callbacks). +Pass your `compute_metrics` function to [KerasMetricCallback](../main_classes/keras_callbacks#transformers.KerasMetricCallback), +and use the [PushToHubCallback](../main_classes/keras_callbacks#transformers.PushToHubCallback) to upload the model: ```py >>> from transformers.keras_callbacks import KerasMetricCallback, PushToHubCallback diff --git a/docs/source/en/tasks/multiple_choice.mdx b/docs/source/en/tasks/multiple_choice.mdx index cafe9d576928..874d22985964 100644 --- a/docs/source/en/tasks/multiple_choice.mdx +++ b/docs/source/en/tasks/multiple_choice.mdx @@ -341,7 +341,7 @@ Configure the model for training with [`compile`](https://keras.io/api/models/mo >>> model.compile(optimizer=optimizer) ``` -The last two things to setup before you start training is to compute the accuracy from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](./main_classes/keras_callbacks). +The last two things to setup before you start training is to compute the accuracy from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](../main_classes/keras_callbacks). Pass your `compute_metrics` function to [`~transformers.KerasMetricCallback`]: diff --git a/docs/source/en/tasks/semantic_segmentation.mdx b/docs/source/en/tasks/semantic_segmentation.mdx index a76837617847..ffaa251688f5 100644 --- a/docs/source/en/tasks/semantic_segmentation.mdx +++ b/docs/source/en/tasks/semantic_segmentation.mdx @@ -412,7 +412,7 @@ Convert your datasets to the `tf.data.Dataset` format using the [`~datasets.Data ... ) ``` -To compute the accuracy from the predictions and push your model to the 🤗 Hub, use [Keras callbacks](./main_classes/keras_callbacks). +To compute the accuracy from the predictions and push your model to the 🤗 Hub, use [Keras callbacks](../main_classes/keras_callbacks). Pass your `compute_metrics` function to [`KerasMetricCallback`], and use the [`PushToHubCallback`] to upload the model: @@ -587,4 +587,4 @@ To visualize the results, load the [dataset color palette](https://github.com/te
Image of bedroom overlaid with segmentation map -
\ No newline at end of file + diff --git a/docs/source/en/tasks/sequence_classification.mdx b/docs/source/en/tasks/sequence_classification.mdx index 20a3f4d14686..b4c57b3fa6f8 100644 --- a/docs/source/en/tasks/sequence_classification.mdx +++ b/docs/source/en/tasks/sequence_classification.mdx @@ -16,7 +16,7 @@ specific language governing permissions and limitations under the License. -Text classification is a common NLP task that assigns a label or class to text. Some of the largest companies run text classification in production for a wide range of practical applications. One of the most popular forms of text classification is sentiment analysis, which assigns a label like 🙂 positive, 🙁 negative, or 😐 neutral to a sequence of text. +Text classification is a common NLP task that assigns a label or class to text. Some of the largest companies run text classification in production for a wide range of practical applications. One of the most popular forms of text classification is sentiment analysis, which assigns a label like 🙂 positive, 🙁 negative, or 😐 neutral to a sequence of text. This guide will show you how to: @@ -69,7 +69,7 @@ Then take a look at an example: } ``` -There are two fields in this dataset: +There are two fields in this dataset: - `text`: the movie review text. - `label`: a value that is either `0` for a negative review or `1` for a positive review. @@ -267,7 +267,7 @@ Configure the model for training with [`compile`](https://keras.io/api/models/mo >>> model.compile(optimizer=optimizer) ``` -The last two things to setup before you start training is to compute the accuracy from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](./main_classes/keras_callbacks). +The last two things to setup before you start training is to compute the accuracy from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](../main_classes/keras_callbacks). Pass your `compute_metrics` function to [`~transformers.KerasMetricCallback`]: diff --git a/docs/source/en/tasks/summarization.mdx b/docs/source/en/tasks/summarization.mdx index 71b1f94d8a97..efb2db60bda3 100644 --- a/docs/source/en/tasks/summarization.mdx +++ b/docs/source/en/tasks/summarization.mdx @@ -19,7 +19,7 @@ specific language governing permissions and limitations under the License. Summarization creates a shorter version of a document or an article that captures all the important information. Along with translation, it is another example of a task that can be formulated as a sequence-to-sequence task. Summarization can be: - Extractive: extract the most relevant information from a document. -- Abstractive: generate new text that captures the most relevant information. +- Abstractive: generate new text that captures the most relevant information. This guide will show you how to: @@ -275,7 +275,7 @@ Configure the model for training with [`compile`](https://keras.io/api/models/mo >>> model.compile(optimizer=optimizer) ``` -The last two things to setup before you start training is to compute the ROUGE score from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](./main_classes/keras_callbacks). +The last two things to setup before you start training is to compute the ROUGE score from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](../main_classes/keras_callbacks). Pass your `compute_metrics` function to [`~transformers.KerasMetricCallback`]: @@ -354,7 +354,7 @@ Tokenize the text and return the `input_ids` as PyTorch tensors: >>> inputs = tokenizer(text, return_tensors="pt").input_ids ``` -Use the [`~transformers.generation_utils.GenerationMixin.generate`] method to create the summarization. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](./main_classes/text_generation) API. +Use the [`~transformers.generation_utils.GenerationMixin.generate`] method to create the summarization. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](../main_classes/text_generation) API. ```py >>> from transformers import AutoModelForSeq2SeqLM @@ -380,7 +380,7 @@ Tokenize the text and return the `input_ids` as TensorFlow tensors: >>> inputs = tokenizer(text, return_tensors="tf").input_ids ``` -Use the [`~transformers.generation_tf_utils.TFGenerationMixin.generate`] method to create the summarization. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](./main_classes/text_generation) API. +Use the [`~transformers.generation_tf_utils.TFGenerationMixin.generate`] method to create the summarization. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](../main_classes/text_generation) API. ```py >>> from transformers import TFAutoModelForSeq2SeqLM @@ -396,4 +396,4 @@ Decode the generated token ids back into text: 'the inflation reduction act lowers prescription drug costs, health care costs, and energy costs. it's the most aggressive action on tackling the climate crisis in american history. it will ask the ultra-wealthy and corporations to pay their fair share.' ``` - \ No newline at end of file + diff --git a/docs/source/en/tasks/token_classification.mdx b/docs/source/en/tasks/token_classification.mdx index a78922fd8c38..7f6032a0ec4e 100644 --- a/docs/source/en/tasks/token_classification.mdx +++ b/docs/source/en/tasks/token_classification.mdx @@ -16,7 +16,7 @@ specific language governing permissions and limitations under the License. -Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization. +Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization. This guide will show you how to: @@ -369,7 +369,7 @@ Configure the model for training with [`compile`](https://keras.io/api/models/mo >>> model.compile(optimizer=optimizer) ``` -The last two things to setup before you start training is to compute the seqeval scores from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](./main_classes/keras_callbacks). +The last two things to setup before you start training is to compute the seqeval scores from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](../main_classes/keras_callbacks). Pass your `compute_metrics` function to [`~transformers.KerasMetricCallback`]: diff --git a/docs/source/en/tasks/translation.mdx b/docs/source/en/tasks/translation.mdx index 6f3d1260d19d..df9df832daf9 100644 --- a/docs/source/en/tasks/translation.mdx +++ b/docs/source/en/tasks/translation.mdx @@ -284,7 +284,7 @@ Configure the model for training with [`compile`](https://keras.io/api/models/mo >>> model.compile(optimizer=optimizer) ``` -The last two things to setup before you start training is to compute the SacreBLEU metric from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](./main_classes/keras_callbacks). +The last two things to setup before you start training is to compute the SacreBLEU metric from the predictions, and provide a way to push your model to the Hub. Both are done by using [Keras callbacks](../main_classes/keras_callbacks). Pass your `compute_metrics` function to [`~transformers.KerasMetricCallback`]: @@ -362,7 +362,7 @@ Tokenize the text and return the `input_ids` as PyTorch tensors: >>> inputs = tokenizer(text, return_tensors="pt").input_ids ``` -Use the [`~transformers.generation_utils.GenerationMixin.generate`] method to create the translation. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](./main_classes/text_generation) API. +Use the [`~transformers.generation_utils.GenerationMixin.generate`] method to create the translation. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](../main_classes/text_generation) API. ```py >>> from transformers import AutoModelForSeq2SeqLM @@ -388,7 +388,7 @@ Tokenize the text and return the `input_ids` as TensorFlow tensors: >>> inputs = tokenizer(text, return_tensors="tf").input_ids ``` -Use the [`~transformers.generation_tf_utils.TFGenerationMixin.generate`] method to create the translation. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](./main_classes/text_generation) API. +Use the [`~transformers.generation_tf_utils.TFGenerationMixin.generate`] method to create the translation. For more details about the different text generation strategies and parameters for controlling generation, check out the [Text Generation](../main_classes/text_generation) API. ```py >>> from transformers import TFAutoModelForSeq2SeqLM @@ -404,4 +404,4 @@ Decode the generated token ids back into text: 'Les lugumes partagent les ressources avec des bactéries fixatrices d'azote.' ``` - \ No newline at end of file + diff --git a/docs/source/es/tasks/summarization.mdx b/docs/source/es/tasks/summarization.mdx index c09c4b0b833a..066ac288b49b 100644 --- a/docs/source/es/tasks/summarization.mdx +++ b/docs/source/es/tasks/summarization.mdx @@ -219,4 +219,4 @@ Para un ejemplo con mayor profundidad de cómo hacer fine-tuning a un modelo par [notebook en PyTorch](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization.ipynb) o a la [notebook en TensorFlow](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/summarization-tf.ipynb). - \ No newline at end of file + From 753db9f58300e9b29cfa01ba662dd79d7aeab860 Mon Sep 17 00:00:00 2001 From: Antoni Viros Date: Mon, 20 Mar 2023 13:54:01 -0400 Subject: [PATCH 141/277] Move torch.compile() wrapping after DDP/FSDP wrapping to ensure correct graph breaks during training (#22279) --- src/transformers/trainer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index c49e004f6e59..7c7d5df0b6a2 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -1361,9 +1361,6 @@ def ipex_optimize_model(self, model, training=False, dtype=torch.float32): return model def _wrap_model(self, model, training=True, dataloader=None): - if self.args.torch_compile: - model = torch.compile(model, backend=self.args.torch_compile_backend, mode=self.args.torch_compile_mode) - if self.args.use_ipex: dtype = torch.bfloat16 if self.use_cpu_amp else torch.float32 model = self.ipex_optimize_model(model, training, dtype=dtype) @@ -1550,6 +1547,11 @@ def patched_optimizer_step(optimizer, barrier=False, optimizer_args={}): **kwargs, ) + # torch.compile() needs to be called after wrapping the model with FSDP or DDP + # to ensure that it accounts for the graph breaks required by those wrappers + if self.args.torch_compile: + model = torch.compile(model, backend=self.args.torch_compile_backend, mode=self.args.torch_compile_mode) + return model def train( From 3feec97ff47a60cc6911c2752d623294a9a0a2f1 Mon Sep 17 00:00:00 2001 From: Maria Khalusova Date: Mon, 20 Mar 2023 14:18:55 -0400 Subject: [PATCH 142/277] Example of pad_to_multiple_of for padding and truncation guide & docstring update (#22278) * added an example of pad_to_multiple_of * make style * addressed feedback --- docs/source/en/pad_truncation.mdx | 1 + src/transformers/tokenization_utils_base.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/en/pad_truncation.mdx b/docs/source/en/pad_truncation.mdx index f848e23bed50..8862e0be008d 100644 --- a/docs/source/en/pad_truncation.mdx +++ b/docs/source/en/pad_truncation.mdx @@ -50,6 +50,7 @@ The following table summarizes the recommended way to setup padding and truncati | | | `tokenizer(batch_sentences, padding='longest')` | | | padding to max model input length | `tokenizer(batch_sentences, padding='max_length')` | | | padding to specific length | `tokenizer(batch_sentences, padding='max_length', max_length=42)` | +| | padding to a multiple of a value | `tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8) | | truncation to max model input length | no padding | `tokenizer(batch_sentences, truncation=True)` or | | | | `tokenizer(batch_sentences, truncation=STRATEGY)` | | | padding to max sequence in batch | `tokenizer(batch_sentences, padding=True, truncation=True)` or | diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index eb52ef0adb28..66164c27788c 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1342,8 +1342,9 @@ def all_special_ids(self) -> List[int]: tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) which it will tokenize. This is useful for NER or token classification. pad_to_multiple_of (`int`, *optional*): - If set will pad the sequence to a multiple of the provided value. This is especially useful to enable - the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta). + If set will pad the sequence to a multiple of the provided value. Requires `padding` to be activated. + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability + `>= 7.5` (Volta). return_tensors (`str` or [`~utils.TensorType`], *optional*): If set, will return tensors instead of list of python integers. Acceptable values are: From 061d16b411c303aef8bcca8a8e91bd5dbf9181cb Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Mon, 20 Mar 2023 20:06:16 +0000 Subject: [PATCH 143/277] Update vision docstring bool masked pos (#22237) * Add bool_masked_pos to forward docstrings * Add note about mask ratio - videomae * Fix up * Fix indenting --- src/transformers/models/beit/modeling_beit.py | 4 ++++ src/transformers/models/beit/modeling_flax_beit.py | 2 +- .../models/data2vec/modeling_data2vec_vision.py | 4 ++++ .../models/data2vec/modeling_tf_data2vec_vision.py | 4 ++++ src/transformers/models/deit/modeling_deit.py | 4 ++++ src/transformers/models/deit/modeling_tf_deit.py | 4 ++++ src/transformers/models/donut/modeling_donut_swin.py | 4 ++++ src/transformers/models/swin/modeling_swin.py | 4 ++++ src/transformers/models/swin/modeling_tf_swin.py | 4 ++++ src/transformers/models/swinv2/modeling_swinv2.py | 4 ++++ src/transformers/models/videomae/modeling_videomae.py | 10 ++++++++++ src/transformers/models/vit/modeling_vit.py | 4 ++++ .../models/vit_hybrid/modeling_vit_hybrid.py | 4 ++++ src/transformers/models/vit_msn/modeling_vit_msn.py | 3 +++ 14 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/beit/modeling_beit.py b/src/transformers/models/beit/modeling_beit.py index 4caf0f478fb6..2b2a06256cf5 100755 --- a/src/transformers/models/beit/modeling_beit.py +++ b/src/transformers/models/beit/modeling_beit.py @@ -659,6 +659,10 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[tuple, BeitModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/beit/modeling_flax_beit.py b/src/transformers/models/beit/modeling_flax_beit.py index 328f75990103..0f0dc809e680 100644 --- a/src/transformers/models/beit/modeling_flax_beit.py +++ b/src/transformers/models/beit/modeling_flax_beit.py @@ -831,7 +831,7 @@ class FlaxBeitForMaskedImageModeling(FlaxBeitPreTrainedModel): FLAX_BEIT_MLM_DOCSTRING = """ bool_masked_pos (`numpy.ndarray` of shape `(batch_size, num_patches)`): - Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Returns: diff --git a/src/transformers/models/data2vec/modeling_data2vec_vision.py b/src/transformers/models/data2vec/modeling_data2vec_vision.py index 42a1edcb6493..d238dd05d216 100644 --- a/src/transformers/models/data2vec/modeling_data2vec_vision.py +++ b/src/transformers/models/data2vec/modeling_data2vec_vision.py @@ -673,6 +673,10 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[tuple, Data2VecVisionModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py b/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py index eca13d433f29..06a6f010dd74 100644 --- a/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py +++ b/src/transformers/models/data2vec/modeling_tf_data2vec_vision.py @@ -907,6 +907,10 @@ def call( return_dict: Optional[bool] = None, training: bool = False, ) -> Union[tuple, TFData2VecVisionModelOutputWithPooling]: + r""" + bool_masked_pos (`tf.Tensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ outputs = self.data2vec_vision( pixel_values=pixel_values, bool_masked_pos=bool_masked_pos, diff --git a/src/transformers/models/deit/modeling_deit.py b/src/transformers/models/deit/modeling_deit.py index f05b16efe7a0..2a37e3ebc1f6 100644 --- a/src/transformers/models/deit/modeling_deit.py +++ b/src/transformers/models/deit/modeling_deit.py @@ -495,6 +495,10 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/deit/modeling_tf_deit.py b/src/transformers/models/deit/modeling_tf_deit.py index 161f2518d068..1bcfd07b38ff 100644 --- a/src/transformers/models/deit/modeling_tf_deit.py +++ b/src/transformers/models/deit/modeling_tf_deit.py @@ -666,6 +666,10 @@ def call( return_dict: Optional[bool] = None, training: bool = False, ): + r""" + bool_masked_pos (`tf.Tensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ outputs = self.deit( pixel_values=pixel_values, bool_masked_pos=bool_masked_pos, diff --git a/src/transformers/models/donut/modeling_donut_swin.py b/src/transformers/models/donut/modeling_donut_swin.py index bb9e863a36c0..65c48eb81f83 100644 --- a/src/transformers/models/donut/modeling_donut_swin.py +++ b/src/transformers/models/donut/modeling_donut_swin.py @@ -911,6 +911,10 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, DonutSwinModelOutput]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/swin/modeling_swin.py b/src/transformers/models/swin/modeling_swin.py index 5f572c23a8ac..17cd0a3d1ad2 100644 --- a/src/transformers/models/swin/modeling_swin.py +++ b/src/transformers/models/swin/modeling_swin.py @@ -976,6 +976,10 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, SwinModelOutput]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/swin/modeling_tf_swin.py b/src/transformers/models/swin/modeling_tf_swin.py index 1022990280a9..e2fdef381329 100644 --- a/src/transformers/models/swin/modeling_tf_swin.py +++ b/src/transformers/models/swin/modeling_tf_swin.py @@ -1208,6 +1208,10 @@ def call( return_dict: Optional[bool] = None, training: bool = False, ) -> Union[TFSwinModelOutput, Tuple[tf.Tensor, ...]]: + r""" + bool_masked_pos (`tf.Tensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/swinv2/modeling_swinv2.py b/src/transformers/models/swinv2/modeling_swinv2.py index 21d61c4e5d65..eff3c8d7946f 100644 --- a/src/transformers/models/swinv2/modeling_swinv2.py +++ b/src/transformers/models/swinv2/modeling_swinv2.py @@ -1055,6 +1055,10 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, Swinv2ModelOutput]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/videomae/modeling_videomae.py b/src/transformers/models/videomae/modeling_videomae.py index 2ec9c01e6f0f..c62d0c4632cb 100644 --- a/src/transformers/models/videomae/modeling_videomae.py +++ b/src/transformers/models/videomae/modeling_videomae.py @@ -571,6 +571,11 @@ def forward( return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutput]: r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Each video in the + batch must have the same number of masked patches. If `None`, then all patches are considered. Sequence + length is `(num_frames // tubelet_size) * (image_size // patch_size) ** 2`. + Returns: Examples: @@ -781,6 +786,11 @@ def forward( return_dict: Optional[bool] = None, ) -> Union[tuple, VideoMAEForPreTrainingOutput]: r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Each video in the + batch must have the same number of masked patches. Sequence length is `(num_frames // tubelet_size) * + (image_size // patch_size) ** 2`. + Returns: Examples: diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index 449eda3ee821..03a9212ff03f 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -549,6 +549,10 @@ def forward( interpolate_pos_encoding: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py b/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py index 7ad3183421bc..5bc9b76a6359 100644 --- a/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py +++ b/src/transformers/models/vit_hybrid/modeling_vit_hybrid.py @@ -573,6 +573,10 @@ def forward( interpolate_pos_encoding: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states diff --git a/src/transformers/models/vit_msn/modeling_vit_msn.py b/src/transformers/models/vit_msn/modeling_vit_msn.py index d2c8547aa0d8..8cba7f5a524f 100644 --- a/src/transformers/models/vit_msn/modeling_vit_msn.py +++ b/src/transformers/models/vit_msn/modeling_vit_msn.py @@ -525,6 +525,9 @@ def forward( return_dict: Optional[bool] = None, ) -> Union[tuple, BaseModelOutput]: r""" + bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*): + Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). + Returns: Examples: From 1703d084e7499646857fba4f319e03b1364a17cc Mon Sep 17 00:00:00 2001 From: Andrei Panferov Date: Tue, 21 Mar 2023 13:16:07 +0300 Subject: [PATCH 144/277] replace_8bit_linear modules_to_not_convert default value fix (#22238) * Fixed modules_to_not_convert default value * Fixed modules_to_not_convert docstring * Update src/transformers/utils/bitsandbytes.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update src/transformers/utils/bitsandbytes.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * ["lm_head"] if modules_to_not_convert is None --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- src/transformers/utils/bitsandbytes.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/transformers/utils/bitsandbytes.py b/src/transformers/utils/bitsandbytes.py index efd9abf6ce50..aee3fec9744a 100644 --- a/src/transformers/utils/bitsandbytes.py +++ b/src/transformers/utils/bitsandbytes.py @@ -84,7 +84,7 @@ class `Int8Params` from `bitsandbytes`. module._parameters[tensor_name] = new_value -def replace_8bit_linear(model, threshold=6.0, modules_to_not_convert="lm_head", current_key_name=None): +def replace_8bit_linear(model, threshold=6.0, modules_to_not_convert=None, current_key_name=None): """ A helper function to replace all `torch.nn.Linear` modules by `bnb.nn.Linear8bit` modules from the `bitsandbytes` library. This will enable running your models using mixed int8 precision as described by the paper `GPT3.int8(): @@ -105,14 +105,15 @@ def replace_8bit_linear(model, threshold=6.0, modules_to_not_convert="lm_head", threshold (`float`, *optional*, defaults to 6.0): `int8_threshold` for outlier detection as described in the formentioned paper. This parameters is set to `6.0` as described by the paper. - modules_to_not_convert (`str`, *optional*, defaults to `lm_head`): - Name of the module to not convert in `Linear8bitLt`. In practice we keep the `lm_head` in full precision + modules_to_not_convert (`List[`str`]`, *optional*, defaults to `["lm_head"]`): + Names of the modules to not convert in `Linear8bitLt`. In practice we keep the `lm_head` in full precision for numerical stability reasons. current_key_name (`List[`str`]`, *optional*): An array to track the current key of the recursion. This is used to check whether the current key (part of it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or `disk`). """ + modules_to_not_convert = ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert for name, module in model.named_children(): if current_key_name is None: current_key_name = [] From 6f9eef8fba34f8d5d30e39c281364d2627856330 Mon Sep 17 00:00:00 2001 From: Gerald Cuder <60609608+gcuder@users.noreply.github.com> Date: Tue, 21 Mar 2023 13:12:57 +0100 Subject: [PATCH 145/277] Fix error in mixed precision training of `TFCvtModel` (#22267) * Make sure CVT can be trained using mixed precision * Add test for keras-fit with mixed-precision * Update tests/models/cvt/test_modeling_tf_cvt.py Co-authored-by: Matt --------- Co-authored-by: gcuder Co-authored-by: Matt --- src/transformers/models/cvt/modeling_tf_cvt.py | 2 +- tests/models/cvt/test_modeling_tf_cvt.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/cvt/modeling_tf_cvt.py b/src/transformers/models/cvt/modeling_tf_cvt.py index 52cc6585a7a5..6ad86071e47d 100644 --- a/src/transformers/models/cvt/modeling_tf_cvt.py +++ b/src/transformers/models/cvt/modeling_tf_cvt.py @@ -93,7 +93,7 @@ def call(self, x: tf.Tensor, training=None): return x keep_prob = 1 - self.drop_prob shape = (tf.shape(x)[0],) + (1,) * (len(tf.shape(x)) - 1) - random_tensor = keep_prob + tf.random.uniform(shape, 0, 1) + random_tensor = keep_prob + tf.random.uniform(shape, 0, 1, dtype=self.compute_dtype) random_tensor = tf.floor(random_tensor) return (x / keep_prob) * random_tensor diff --git a/tests/models/cvt/test_modeling_tf_cvt.py b/tests/models/cvt/test_modeling_tf_cvt.py index 4605f6782bde..484bd295d172 100644 --- a/tests/models/cvt/test_modeling_tf_cvt.py +++ b/tests/models/cvt/test_modeling_tf_cvt.py @@ -186,6 +186,12 @@ def test_dataset_conversion(self): def test_keras_fit(self): super().test_keras_fit() + def test_keras_fit_mixed_precision(self): + policy = tf.keras.mixed_precision.Policy("mixed_float16") + tf.keras.mixed_precision.set_global_policy(policy) + super().test_keras_fit() + tf.keras.mixed_precision.set_global_policy("float32") + def test_forward_signature(self): config, _ = self.model_tester.prepare_config_and_inputs_for_common() From afc258defb1b65560bf16a87d24caf0f0cd4536e Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 21 Mar 2023 13:27:30 +0100 Subject: [PATCH 146/277] More doctests (#22268) * all doctests * Skip failed tests --------- Co-authored-by: ydshieh --- utils/documentation_tests.txt | 222 +++++++++++++++++++++++++++++++++- 1 file changed, 218 insertions(+), 4 deletions(-) diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 8b622bf778dc..7f3535eb5579 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -162,7 +162,6 @@ src/transformers/models/roberta_prelayernorm/configuration_roberta_prelayernorm. src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py src/transformers/models/roberta_prelayernorm/modeling_tf_roberta_prelayernorm.py src/transformers/models/roc_bert/modeling_roc_bert.py -src/transformers/models/roc_bert/tokenization_roc_bert.py src/transformers/models/segformer/modeling_segformer.py src/transformers/models/sew/configuration_sew.py src/transformers/models/sew/modeling_sew.py @@ -174,7 +173,6 @@ src/transformers/models/speech_to_text/modeling_speech_to_text.py src/transformers/models/speech_to_text_2/configuration_speech_to_text_2.py src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py src/transformers/models/speecht5/modeling_speecht5.py -src/transformers/models/speecht5/tokenization_speecht5.py src/transformers/models/segformer/modeling_tf_segformer.py src/transformers/models/squeezebert/configuration_squeezebert.py src/transformers/models/swin/configuration_swin.py @@ -209,10 +207,8 @@ src/transformers/models/vit_msn/modeling_vit_msn.py src/transformers/models/visual_bert/configuration_visual_bert.py src/transformers/models/wav2vec2/configuration_wav2vec2.py src/transformers/models/wav2vec2/modeling_wav2vec2.py -src/transformers/models/wav2vec2/tokenization_wav2vec2.py src/transformers/models/wav2vec2_conformer/configuration_wav2vec2_conformer.py src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py -src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py src/transformers/models/wavlm/configuration_wavlm.py src/transformers/models/wavlm/modeling_wavlm.py src/transformers/models/whisper/configuration_whisper.py @@ -230,3 +226,221 @@ src/transformers/models/x_clip/modeling_x_clip.py src/transformers/models/yoso/configuration_yoso.py src/transformers/models/timesformer/configuration_timesformer.py src/transformers/pipelines/ +src/transformers/models/albert/tokenization_albert.py +src/transformers/models/albert/tokenization_albert_fast.py +src/transformers/models/barthez/tokenization_barthez.py +src/transformers/models/barthez/tokenization_barthez_fast.py +src/transformers/models/bartpho/tokenization_bartpho.py +src/transformers/models/bert/tokenization_bert.py +src/transformers/models/bert/tokenization_bert_fast.py +src/transformers/models/bert/tokenization_bert_tf.py +src/transformers/models/bert_generation/tokenization_bert_generation.py +src/transformers/models/bert_japanese/tokenization_bert_japanese.py +src/transformers/models/big_bird/tokenization_big_bird.py +src/transformers/models/big_bird/tokenization_big_bird_fast.py +src/transformers/models/biogpt/tokenization_biogpt.py +src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py +src/transformers/models/blenderbot_small/tokenization_blenderbot_small_fast.py +src/transformers/models/byt5/tokenization_byt5.py +src/transformers/models/camembert/tokenization_camembert.py +src/transformers/models/camembert/tokenization_camembert_fast.py +src/transformers/models/canine/tokenization_canine.py +src/transformers/models/clip/tokenization_clip.py +src/transformers/models/clip/tokenization_clip_fast.py +src/transformers/models/convbert/tokenization_convbert.py +src/transformers/models/convbert/tokenization_convbert_fast.py +src/transformers/models/cpm/tokenization_cpm.py +src/transformers/models/cpm/tokenization_cpm_fast.py +src/transformers/models/ctrl/tokenization_ctrl.py +src/transformers/models/deberta_v2/tokenization_deberta_v2.py +src/transformers/models/deberta_v2/tokenization_deberta_v2_fast.py +src/transformers/models/distilbert/tokenization_distilbert.py +src/transformers/models/distilbert/tokenization_distilbert_fast.py +src/transformers/models/electra/tokenization_electra.py +src/transformers/models/electra/tokenization_electra_fast.py +src/transformers/models/ernie_m/tokenization_ernie_m.py +src/transformers/models/esm/tokenization_esm.py +src/transformers/models/flaubert/tokenization_flaubert.py +src/transformers/models/fnet/tokenization_fnet.py +src/transformers/models/fnet/tokenization_fnet_fast.py +src/transformers/models/fsmt/tokenization_fsmt.py +src/transformers/models/funnel/tokenization_funnel.py +src/transformers/models/funnel/tokenization_funnel_fast.py +src/transformers/models/gpt2/tokenization_gpt2_tf.py +src/transformers/models/gpt_neox_japanese/tokenization_gpt_neox_japanese.py +src/transformers/models/herbert/tokenization_herbert.py +src/transformers/models/herbert/tokenization_herbert_fast.py +src/transformers/models/jukebox/tokenization_jukebox.py +src/transformers/models/layoutlm/tokenization_layoutlm.py +src/transformers/models/layoutlm/tokenization_layoutlm_fast.py +src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py +src/transformers/models/layoutlmv2/tokenization_layoutlmv2_fast.py +src/transformers/models/layoutlmv3/tokenization_layoutlmv3.py +src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py +src/transformers/models/layoutxlm/tokenization_layoutxlm.py +src/transformers/models/layoutxlm/tokenization_layoutxlm_fast.py +src/transformers/models/llama/tokenization_llama.py +src/transformers/models/lxmert/tokenization_lxmert.py +src/transformers/models/lxmert/tokenization_lxmert_fast.py +src/transformers/models/markuplm/tokenization_markuplm.py +src/transformers/models/markuplm/tokenization_markuplm_fast.py +src/transformers/models/mbart/tokenization_mbart.py +src/transformers/models/mbart/tokenization_mbart_fast.py +src/transformers/models/mbart50/tokenization_mbart50.py +src/transformers/models/mbart50/tokenization_mbart50_fast.py +src/transformers/models/mgp_str/tokenization_mgp_str.py +src/transformers/models/mluke/tokenization_mluke.py +src/transformers/models/mobilebert/tokenization_mobilebert.py +src/transformers/models/mobilebert/tokenization_mobilebert_fast.py +src/transformers/models/mpnet/tokenization_mpnet.py +src/transformers/models/mpnet/tokenization_mpnet_fast.py +src/transformers/models/nllb/tokenization_nllb.py +src/transformers/models/nllb/tokenization_nllb_fast.py +src/transformers/models/openai/tokenization_openai.py +src/transformers/models/openai/tokenization_openai_fast.py +src/transformers/models/pegasus/tokenization_pegasus.py +src/transformers/models/pegasus/tokenization_pegasus_fast.py +src/transformers/models/perceiver/tokenization_perceiver.py +src/transformers/models/phobert/tokenization_phobert.py +src/transformers/models/plbart/tokenization_plbart.py +src/transformers/models/prophetnet/tokenization_prophetnet.py +src/transformers/models/rag/tokenization_rag.py +src/transformers/models/realm/tokenization_realm.py +src/transformers/models/realm/tokenization_realm_fast.py +src/transformers/models/reformer/tokenization_reformer.py +src/transformers/models/reformer/tokenization_reformer_fast.py +src/transformers/models/rembert/tokenization_rembert.py +src/transformers/models/rembert/tokenization_rembert_fast.py +src/transformers/models/retribert/tokenization_retribert.py +src/transformers/models/retribert/tokenization_retribert_fast.py +src/transformers/models/roc_bert/tokenization_roc_bert.py +src/transformers/models/roformer/tokenization_utils.py +src/transformers/models/speecht5/tokenization_speecht5.py +src/transformers/models/speech_to_text/tokenization_speech_to_text.py +src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py +src/transformers/models/splinter/tokenization_splinter.py +src/transformers/models/splinter/tokenization_splinter_fast.py +src/transformers/models/squeezebert/tokenization_squeezebert.py +src/transformers/models/squeezebert/tokenization_squeezebert_fast.py +src/transformers/models/t5/tokenization_t5.py +src/transformers/models/t5/tokenization_t5_fast.py +src/transformers/models/tapas/tokenization_tapas.py +src/transformers/models/tapex/tokenization_tapex.py +src/transformers/models/wav2vec2/tokenization_wav2vec2.py +src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py +src/transformers/models/whisper/tokenization_whisper.py +src/transformers/models/whisper/tokenization_whisper_fast.py +src/transformers/models/xglm/tokenization_xglm.py +src/transformers/models/xglm/tokenization_xglm_fast.py +src/transformers/models/xlm/tokenization_xlm.py +src/transformers/models/xlm_prophetnet/tokenization_xlm_prophetnet.py +src/transformers/models/xlm_roberta/tokenization_xlm_roberta.py +src/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.py +src/transformers/models/xlnet/tokenization_xlnet.py +src/transformers/models/xlnet/tokenization_xlnet_fast.py +src/transformers/models/beit/image_processing_beit.py +src/transformers/models/bit/image_processing_bit.py +src/transformers/models/blip/image_processing_blip.py +src/transformers/models/bridgetower/image_processing_bridgetower.py +src/transformers/models/chinese_clip/image_processing_chinese_clip.py +src/transformers/models/clip/image_processing_clip.py +src/transformers/models/conditional_detr/image_processing_conditional_detr.py +src/transformers/models/convnext/image_processing_convnext.py +src/transformers/models/deformable_detr/image_processing_deformable_detr.py +src/transformers/models/deit/image_processing_deit.py +src/transformers/models/deta/image_processing_deta.py +src/transformers/models/detr/image_processing_detr.py +src/transformers/models/donut/image_processing_donut.py +src/transformers/models/dpt/image_processing_dpt.py +src/transformers/models/efficientformer/image_processing_efficientformer.py +src/transformers/models/efficientnet/image_processing_efficientnet.py +src/transformers/models/flava/image_processing_flava.py +src/transformers/models/glpn/image_processing_glpn.py +src/transformers/models/imagegpt/image_processing_imagegpt.py +src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py +src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py +src/transformers/models/levit/image_processing_levit.py +src/transformers/models/mask2former/image_processing_mask2former.py +src/transformers/models/maskformer/image_processing_maskformer.py +src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py +src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py +src/transformers/models/mobilevit/image_processing_mobilevit.py +src/transformers/models/oneformer/image_processing_oneformer.py +src/transformers/models/owlvit/image_processing_owlvit.py +src/transformers/models/perceiver/image_processing_perceiver.py +src/transformers/models/poolformer/image_processing_poolformer.py +src/transformers/models/segformer/image_processing_segformer.py +src/transformers/models/swin2sr/image_processing_swin2sr.py +src/transformers/models/tvlt/image_processing_tvlt.py +src/transformers/models/videomae/image_processing_videomae.py +src/transformers/models/vilt/image_processing_vilt.py +src/transformers/models/vit/image_processing_vit.py +src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py +src/transformers/models/yolos/image_processing_yolos.py +src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py +src/transformers/models/beit/feature_extraction_beit.py +src/transformers/models/chinese_clip/feature_extraction_chinese_clip.py +src/transformers/models/clap/feature_extraction_clap.py +src/transformers/models/clip/feature_extraction_clip.py +src/transformers/models/conditional_detr/feature_extraction_conditional_detr.py +src/transformers/models/convnext/feature_extraction_convnext.py +src/transformers/models/deformable_detr/feature_extraction_deformable_detr.py +src/transformers/models/deit/feature_extraction_deit.py +src/transformers/models/detr/feature_extraction_detr.py +src/transformers/models/donut/feature_extraction_donut.py +src/transformers/models/dpt/feature_extraction_dpt.py +src/transformers/models/flava/feature_extraction_flava.py +src/transformers/models/glpn/feature_extraction_glpn.py +src/transformers/models/imagegpt/feature_extraction_imagegpt.py +src/transformers/models/layoutlmv2/feature_extraction_layoutlmv2.py +src/transformers/models/layoutlmv3/feature_extraction_layoutlmv3.py +src/transformers/models/levit/feature_extraction_levit.py +src/transformers/models/maskformer/feature_extraction_maskformer.py +src/transformers/models/mctct/feature_extraction_mctct.py +src/transformers/models/mobilenet_v1/feature_extraction_mobilenet_v1.py +src/transformers/models/mobilenet_v2/feature_extraction_mobilenet_v2.py +src/transformers/models/mobilevit/feature_extraction_mobilevit.py +src/transformers/models/owlvit/feature_extraction_owlvit.py +src/transformers/models/perceiver/feature_extraction_perceiver.py +src/transformers/models/poolformer/feature_extraction_poolformer.py +src/transformers/models/segformer/feature_extraction_segformer.py +src/transformers/models/speecht5/feature_extraction_speecht5.py +src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py +src/transformers/models/tvlt/feature_extraction_tvlt.py +src/transformers/models/videomae/feature_extraction_videomae.py +src/transformers/models/vilt/feature_extraction_vilt.py +src/transformers/models/vit/feature_extraction_vit.py +src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py +src/transformers/models/whisper/feature_extraction_whisper.py +src/transformers/models/yolos/feature_extraction_yolos.py +src/transformers/models/align/processing_align.py +src/transformers/models/altclip/processing_altclip.py +src/transformers/models/blip/processing_blip.py +src/transformers/models/blip_2/processing_blip_2.py +src/transformers/models/bridgetower/processing_bridgetower.py +src/transformers/models/chinese_clip/processing_chinese_clip.py +src/transformers/models/clap/processing_clap.py +src/transformers/models/clip/processing_clip.py +src/transformers/models/clipseg/processing_clipseg.py +src/transformers/models/donut/processing_donut.py +src/transformers/models/flava/processing_flava.py +src/transformers/models/git/processing_git.py +src/transformers/models/layoutlmv2/processing_layoutlmv2.py +src/transformers/models/layoutlmv3/processing_layoutlmv3.py +src/transformers/models/layoutxlm/processing_layoutxlm.py +src/transformers/models/markuplm/processing_markuplm.py +src/transformers/models/mctct/processing_mctct.py +src/transformers/models/mgp_str/processing_mgp_str.py +src/transformers/models/oneformer/processing_oneformer.py +src/transformers/models/owlvit/processing_owlvit.py +src/transformers/models/speecht5/processing_speecht5.py +src/transformers/models/speech_to_text/processing_speech_to_text.py +src/transformers/models/speech_to_text_2/processing_speech_to_text_2.py +src/transformers/models/trocr/processing_trocr.py +src/transformers/models/tvlt/processing_tvlt.py +src/transformers/models/vilt/processing_vilt.py +src/transformers/models/vision_text_dual_encoder/processing_vision_text_dual_encoder.py +src/transformers/models/wav2vec2/processing_wav2vec2.py +src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py +src/transformers/models/whisper/processing_whisper.py +src/transformers/models/x_clip/processing_x_clip.py From b9955aae3063cdc6dd8682dd0507a0e5591f6419 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:16:17 +0100 Subject: [PATCH 147/277] fix more doctests (#22292) * fix more doctests * fix style --------- Co-authored-by: ydshieh --- .../models/bart/tokenization_bart.py | 8 ++++--- .../models/bart/tokenization_bart_fast.py | 8 ++++--- .../blenderbot/tokenization_blenderbot.py | 8 ++++--- .../tokenization_blenderbot_fast.py | 8 ++++--- .../models/bloom/tokenization_bloom_fast.py | 12 ++++++---- .../models/codegen/tokenization_codegen.py | 8 ++++--- .../codegen/tokenization_codegen_fast.py | 8 ++++--- .../models/deberta/tokenization_deberta.py | 12 ++++++---- .../deberta/tokenization_deberta_fast.py | 12 ++++++---- .../models/gpt2/tokenization_gpt2.py | 8 ++++--- .../models/gpt2/tokenization_gpt2_fast.py | 8 ++++--- .../gpt_neox/tokenization_gpt_neox_fast.py | 8 ++++--- .../models/gpt_sw3/tokenization_gpt_sw3.py | 5 ++-- .../models/jukebox/tokenization_jukebox.py | 6 ++--- .../models/led/tokenization_led.py | 8 ++++--- .../models/led/tokenization_led_fast.py | 8 ++++--- .../longformer/tokenization_longformer.py | 8 ++++--- .../tokenization_longformer_fast.py | 8 ++++--- .../models/luke/tokenization_luke.py | 8 ++++--- .../models/mvp/tokenization_mvp.py | 8 ++++--- .../models/mvp/tokenization_mvp_fast.py | 8 ++++--- .../models/roberta/tokenization_roberta.py | 8 ++++--- .../roberta/tokenization_roberta_fast.py | 8 ++++--- utils/documentation_tests.txt | 23 +++++++++++++++++++ 24 files changed, 140 insertions(+), 74 deletions(-) diff --git a/src/transformers/models/bart/tokenization_bart.py b/src/transformers/models/bart/tokenization_bart.py index 5dc93578109f..22ee1a0db614 100644 --- a/src/transformers/models/bart/tokenization_bart.py +++ b/src/transformers/models/bart/tokenization_bart.py @@ -105,12 +105,14 @@ class BartTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import BartTokenizer + >>> tokenizer = BartTokenizer.from_pretrained("facebook/bart-base") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/bart/tokenization_bart_fast.py b/src/transformers/models/bart/tokenization_bart_fast.py index 6d6e29986be4..f05ed1b7a82d 100644 --- a/src/transformers/models/bart/tokenization_bart_fast.py +++ b/src/transformers/models/bart/tokenization_bart_fast.py @@ -75,12 +75,14 @@ class BartTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import BartTokenizerFast + >>> tokenizer = BartTokenizerFast.from_pretrained("facebook/bart-base") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot.py b/src/transformers/models/blenderbot/tokenization_blenderbot.py index 208ced46bc2d..cb4a33a3c28b 100644 --- a/src/transformers/models/blenderbot/tokenization_blenderbot.py +++ b/src/transformers/models/blenderbot/tokenization_blenderbot.py @@ -96,13 +96,15 @@ class BlenderbotTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import BlenderbotTokenizer + >>> tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B") >>> tokenizer.add_prefix_space = False - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [47, 921, 86, 1085, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [6950, 1085, 2] ``` diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py b/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py index 7c4e060e5d20..4737e92617c7 100644 --- a/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py +++ b/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py @@ -55,12 +55,14 @@ class BlenderbotTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import BlenderbotTokenizerFast + >>> tokenizer = BlenderbotTokenizerFast.from_pretrained("facebook/blenderbot-3B") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [6950, 1085, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [6950, 1085, 2] ``` diff --git a/src/transformers/models/bloom/tokenization_bloom_fast.py b/src/transformers/models/bloom/tokenization_bloom_fast.py index 1c8efb10cb6c..800c73f0250b 100644 --- a/src/transformers/models/bloom/tokenization_bloom_fast.py +++ b/src/transformers/models/bloom/tokenization_bloom_fast.py @@ -54,13 +54,15 @@ class BloomTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import BloomTokenizerFast + >>> tokenizer = BloomTokenizerFast.from_pretrained("bigscience/bloom") - >>> tokenizer("Hello world")['input_ids'] - [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] - [18435, 995] + >>> tokenizer("Hello world")["input_ids"] + [59414, 8876] + + >>> tokenizer(" Hello world")["input_ids"] + [86153, 8876] ``` You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer, but since diff --git a/src/transformers/models/codegen/tokenization_codegen.py b/src/transformers/models/codegen/tokenization_codegen.py index c09a816bfbab..0c01e63471dc 100644 --- a/src/transformers/models/codegen/tokenization_codegen.py +++ b/src/transformers/models/codegen/tokenization_codegen.py @@ -102,12 +102,14 @@ class CodeGenTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import CodeGenTokenizer + >>> tokenizer = CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [18435, 995] ``` diff --git a/src/transformers/models/codegen/tokenization_codegen_fast.py b/src/transformers/models/codegen/tokenization_codegen_fast.py index 332f0ed934ac..83e5e30d0b0b 100644 --- a/src/transformers/models/codegen/tokenization_codegen_fast.py +++ b/src/transformers/models/codegen/tokenization_codegen_fast.py @@ -68,12 +68,14 @@ class CodeGenTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import CodeGenTokenizerFast + >>> tokenizer = CodeGenTokenizerFast.from_pretrained("Salesforce/codegen-350M-mono") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [18435, 995] ``` diff --git a/src/transformers/models/deberta/tokenization_deberta.py b/src/transformers/models/deberta/tokenization_deberta.py index bcaaaa442117..8a778a947cfb 100644 --- a/src/transformers/models/deberta/tokenization_deberta.py +++ b/src/transformers/models/deberta/tokenization_deberta.py @@ -116,13 +116,15 @@ class DebertaTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import DebertaTokenizer + >>> tokenizer = DebertaTokenizer.from_pretrained("microsoft/deberta-base") - >>> tokenizer("Hello world")['input_ids'] - [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] - [18435, 995] + >>> tokenizer("Hello world")["input_ids"] + [1, 31414, 232, 2] + + >>> tokenizer(" Hello world")["input_ids"] + [1, 20920, 232, 2] ``` You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you diff --git a/src/transformers/models/deberta/tokenization_deberta_fast.py b/src/transformers/models/deberta/tokenization_deberta_fast.py index 959bcae47011..c05cf257611e 100644 --- a/src/transformers/models/deberta/tokenization_deberta_fast.py +++ b/src/transformers/models/deberta/tokenization_deberta_fast.py @@ -79,13 +79,15 @@ class DebertaTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import DebertaTokenizerFast + >>> tokenizer = DebertaTokenizerFast.from_pretrained("microsoft/deberta-base") - >>> tokenizer("Hello world")['input_ids'] - [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] - [18435, 995] + >>> tokenizer("Hello world")["input_ids"] + [1, 31414, 232, 2] + + >>> tokenizer(" Hello world")["input_ids"] + [1, 20920, 232, 2] ``` You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer, but since diff --git a/src/transformers/models/gpt2/tokenization_gpt2.py b/src/transformers/models/gpt2/tokenization_gpt2.py index c462e45d01e3..9a8ce3a4fabd 100644 --- a/src/transformers/models/gpt2/tokenization_gpt2.py +++ b/src/transformers/models/gpt2/tokenization_gpt2.py @@ -108,12 +108,14 @@ class GPT2Tokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import GPT2Tokenizer + >>> tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [18435, 995] ``` diff --git a/src/transformers/models/gpt2/tokenization_gpt2_fast.py b/src/transformers/models/gpt2/tokenization_gpt2_fast.py index 7d7500ee9cca..cf2b8b2cb22c 100644 --- a/src/transformers/models/gpt2/tokenization_gpt2_fast.py +++ b/src/transformers/models/gpt2/tokenization_gpt2_fast.py @@ -75,12 +75,14 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import GPT2TokenizerFast + >>> tokenizer = GPT2TokenizerFast.from_pretrained("gpt2") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [18435, 995] ``` diff --git a/src/transformers/models/gpt_neox/tokenization_gpt_neox_fast.py b/src/transformers/models/gpt_neox/tokenization_gpt_neox_fast.py index 1d4c1cec3a75..570b2abaa49f 100644 --- a/src/transformers/models/gpt_neox/tokenization_gpt_neox_fast.py +++ b/src/transformers/models/gpt_neox/tokenization_gpt_neox_fast.py @@ -49,12 +49,14 @@ class GPTNeoXTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import GPTNeoXTokenizerFast + >>> tokenizer = GPTNeoXTokenizerFast.from_pretrained("gpt2") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [15496, 995] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [18435, 995] ``` diff --git a/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py b/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py index f982c5b6b171..422cb0722083 100644 --- a/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py +++ b/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py @@ -47,10 +47,11 @@ class GPTSw3Tokenizer(PreTrainedTokenizer): this superclass for more information regarding those methods. Example usage: - ``` + ```python >>> from transformers import GPTSw3Tokenizer + >>> tokenizer = GPTSw3Tokenizer.from_pretrained("AI-Sweden/gpt-sw3-126m") - >>> tokenizer("Svenska är kul!")['input_ids'] + >>> tokenizer("Svenska är kul!")["input_ids"] [1814, 377, 3617, 63504] ``` diff --git a/src/transformers/models/jukebox/tokenization_jukebox.py b/src/transformers/models/jukebox/tokenization_jukebox.py index 63399adf1624..9a45f6bd6555 100644 --- a/src/transformers/models/jukebox/tokenization_jukebox.py +++ b/src/transformers/models/jukebox/tokenization_jukebox.py @@ -68,13 +68,13 @@ class JukeboxTokenizer(PreTrainedTokenizer): as the conditioning of the model can be done on the three different queries. If None is provided, defaults values will be used.: Depending on the number of genres on which the model should be conditioned (`n_genres`). - ``` + ```python >>> from transformers import JukeboxTokenizer + >>> tokenizer = JukeboxTokenizer.from_pretrained("openai/jukebox-1b-lyrics") - >>> tokenizer("Alan Jackson", "Country Rock", "old town road")['input_ids'] + >>> tokenizer("Alan Jackson", "Country Rock", "old town road")["input_ids"] [tensor([[ 0, 0, 0, 6785, 546, 41, 38, 30, 76, 46, 41, 49, 40, 76, 44, 41, 27, 30]]), tensor([[ 0, 0, 0, 145, 0]]), tensor([[ 0, 0, 0, 145, 0]])] - ``` You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you diff --git a/src/transformers/models/led/tokenization_led.py b/src/transformers/models/led/tokenization_led.py index 5b22701a2262..1cdb52430117 100644 --- a/src/transformers/models/led/tokenization_led.py +++ b/src/transformers/models/led/tokenization_led.py @@ -97,12 +97,14 @@ class LEDTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import LEDTokenizer + >>> tokenizer = LEDTokenizer.from_pretrained("allenai/led-base-16384") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/led/tokenization_led_fast.py b/src/transformers/models/led/tokenization_led_fast.py index 153b32c2967b..51b8ab4aaaf0 100644 --- a/src/transformers/models/led/tokenization_led_fast.py +++ b/src/transformers/models/led/tokenization_led_fast.py @@ -55,12 +55,14 @@ class LEDTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import LEDTokenizerFast + >>> tokenizer = LEDTokenizerFast.from_pretrained("allenai/led-base-16384") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/longformer/tokenization_longformer.py b/src/transformers/models/longformer/tokenization_longformer.py index 69bc50595387..fea949658abc 100644 --- a/src/transformers/models/longformer/tokenization_longformer.py +++ b/src/transformers/models/longformer/tokenization_longformer.py @@ -120,12 +120,14 @@ class LongformerTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import LongformerTokenizer + >>> tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/longformer/tokenization_longformer_fast.py b/src/transformers/models/longformer/tokenization_longformer_fast.py index dfe1b08e1458..1460f2f2cc2f 100644 --- a/src/transformers/models/longformer/tokenization_longformer_fast.py +++ b/src/transformers/models/longformer/tokenization_longformer_fast.py @@ -96,12 +96,14 @@ class LongformerTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import LongformerTokenizerFast + >>> tokenizer = LongformerTokenizerFast.from_pretrained("allenai/longformer-base-4096") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/luke/tokenization_luke.py b/src/transformers/models/luke/tokenization_luke.py index 89fb9b63e86e..8b47ced1d317 100644 --- a/src/transformers/models/luke/tokenization_luke.py +++ b/src/transformers/models/luke/tokenization_luke.py @@ -197,12 +197,14 @@ class LukeTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import LukeTokenizer + >>> tokenizer = LukeTokenizer.from_pretrained("studio-ousia/luke-base") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/mvp/tokenization_mvp.py b/src/transformers/models/mvp/tokenization_mvp.py index 98d373188e05..2d497c23d130 100644 --- a/src/transformers/models/mvp/tokenization_mvp.py +++ b/src/transformers/models/mvp/tokenization_mvp.py @@ -93,12 +93,14 @@ class MvpTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import MvpTokenizer + >>> tokenizer = MvpTokenizer.from_pretrained("RUCAIBox/mvp") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/mvp/tokenization_mvp_fast.py b/src/transformers/models/mvp/tokenization_mvp_fast.py index 28dd1ea942df..fd6abd170020 100644 --- a/src/transformers/models/mvp/tokenization_mvp_fast.py +++ b/src/transformers/models/mvp/tokenization_mvp_fast.py @@ -58,12 +58,14 @@ class MvpTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import MvpTokenizerFast + >>> tokenizer = MvpTokenizerFast.from_pretrained("RUCAIBox/mvp") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/roberta/tokenization_roberta.py b/src/transformers/models/roberta/tokenization_roberta.py index e8d4a751bc19..24b9748c3d37 100644 --- a/src/transformers/models/roberta/tokenization_roberta.py +++ b/src/transformers/models/roberta/tokenization_roberta.py @@ -111,12 +111,14 @@ class RobertaTokenizer(PreTrainedTokenizer): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import RobertaTokenizer + >>> tokenizer = RobertaTokenizer.from_pretrained("roberta-base") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/src/transformers/models/roberta/tokenization_roberta_fast.py b/src/transformers/models/roberta/tokenization_roberta_fast.py index 32cc66d750b0..c2c479da0964 100644 --- a/src/transformers/models/roberta/tokenization_roberta_fast.py +++ b/src/transformers/models/roberta/tokenization_roberta_fast.py @@ -81,12 +81,14 @@ class RobertaTokenizerFast(PreTrainedTokenizerFast): This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not: - ``` + ```python >>> from transformers import RobertaTokenizerFast + >>> tokenizer = RobertaTokenizerFast.from_pretrained("roberta-base") - >>> tokenizer("Hello world")['input_ids'] + >>> tokenizer("Hello world")["input_ids"] [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] + + >>> tokenizer(" Hello world")["input_ids"] [0, 20920, 232, 2] ``` diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 7f3535eb5579..bfd87a5c0be1 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -444,3 +444,26 @@ src/transformers/models/wav2vec2/processing_wav2vec2.py src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py src/transformers/models/whisper/processing_whisper.py src/transformers/models/x_clip/processing_x_clip.py +src/transformers/models/bart/tokenization_bart.py +src/transformers/models/bart/tokenization_bart_fast.py +src/transformers/models/blenderbot/tokenization_blenderbot.py +src/transformers/models/blenderbot/tokenization_blenderbot_fast.py +src/transformers/models/bloom/tokenization_bloom_fast.py +src/transformers/models/codegen/tokenization_codegen.py +src/transformers/models/codegen/tokenization_codegen_fast.py +src/transformers/models/deberta/tokenization_deberta.py +src/transformers/models/deberta/tokenization_deberta_fast.py +src/transformers/models/gpt2/tokenization_gpt2.py +src/transformers/models/gpt2/tokenization_gpt2_fast.py +src/transformers/models/gpt_neox/tokenization_gpt_neox_fast.py +src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py +src/transformers/models/jukebox/tokenization_jukebox.py +src/transformers/models/led/tokenization_led.py +src/transformers/models/led/tokenization_led_fast.py +src/transformers/models/longformer/tokenization_longformer.py +src/transformers/models/longformer/tokenization_longformer_fast.py +src/transformers/models/luke/tokenization_luke.py +src/transformers/models/mvp/tokenization_mvp.py +src/transformers/models/mvp/tokenization_mvp_fast.py +src/transformers/models/roberta/tokenization_roberta.py +src/transformers/models/roberta/tokenization_roberta_fast.py From f282b2d56720b118d05b0fe478afc72e55ba7641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Davide=20Gazz=C3=A8?= Date: Tue, 21 Mar 2023 18:07:30 +0100 Subject: [PATCH 148/277] Add translation perf_infer_gpu_one for it (#22296) Add translation --- docs/source/it/_toctree.yml | 2 + docs/source/it/perf_infer_gpu_one.mdx | 108 ++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 docs/source/it/perf_infer_gpu_one.mdx diff --git a/docs/source/it/_toctree.yml b/docs/source/it/_toctree.yml index e0920248fffa..630ade2212d9 100644 --- a/docs/source/it/_toctree.yml +++ b/docs/source/it/_toctree.yml @@ -39,6 +39,8 @@ title: Addestramento efficiente su multiple CPU - local: perf_infer_cpu title: Inferenza Efficiente su CPU + - local: perf_infer_gpu_one + title: Inferenza su una GPU - local: big_models title: Istanziare un big model - local: migration diff --git a/docs/source/it/perf_infer_gpu_one.mdx b/docs/source/it/perf_infer_gpu_one.mdx new file mode 100644 index 000000000000..60df05515351 --- /dev/null +++ b/docs/source/it/perf_infer_gpu_one.mdx @@ -0,0 +1,108 @@ + + +# Inferenza efficiente su GPU singola + +Questo documento sarà presto completato con informazioni su come effetture l'inferenza su una singola GPU. Nel frattempo è possibile consultare [la guida per l'addestramento su una singola GPU](perf_train_gpu_one) e [la guida per l'inferenza su CPU](perf_infer_cpu). + +## `BetterTransformer` per l'inferenza più veloce + +Abbiamo recentemente integrato `BetterTransformer` per velocizzare l'inferenza su GPU per modelli di testo, immagini e audio. Per maggiori dettagli, consultare la documentazione su questa integrazione [qui](https://huggingface.co/docs/optimum/bettertransformer/overview). + +## Integrazione di `bitsandbytes` per Int8 mixed-precision matrix decomposition + + + +Nota che questa funzione può essere utilizzata anche nelle configurazioni multi GPU. + + + +Dal paper [`LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale`](https://arxiv.org/abs/2208.07339), noi supportiamo l'integrazione di Hugging Face per tutti i modelli dell'Hub con poche righe di codice. +Il metodo `nn.Linear` riduce la dimensione di 2 per i pesi `float16` e `bfloat16` e di 4 per i pesi `float32`, con un impatto quasi nullo sulla qualità, operando sugli outlier in half-precision. + +![HFxbitsandbytes.png](https://s3.amazonaws.com/moonup/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png) + +Il metodo Int8 mixed-precision matrix decomposition funziona separando la moltiplicazione tra matrici in due flussi: (1) una matrice di flusso di outlier di caratteristiche sistematiche moltiplicata in fp16, (2) in flusso regolare di moltiplicazione di matrici int8 (99,9%). Con questo metodo, è possibile effettutare inferenza int8 per modelli molto grandi senza degrado predittivo. +Per maggiori dettagli sul metodo, consultare il [paper](https://arxiv.org/abs/2208.07339) o il nostro [blogpost sull'integrazione](https://huggingface.co/blog/hf-bitsandbytes-integration). + +![MixedInt8.gif](https://s3.amazonaws.com/moonup/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif) + +Nota che è necessaria una GPU per eseguire modelli di tipo mixed-8bit, poiché i kernel sono stati compilati solo per le GPU. Prima di utilizzare questa funzione, assicurarsi di disporre di memoria sufficiente sulla GPU per memorizzare un quarto del modello (o la metà se i pesi del modello sono in mezza precisione). +Di seguito sono riportate alcune note per aiutarvi a utilizzare questo modulo, oppure seguite le dimostrazioni su [Google colab](#colab-demos). + +### Requisiti + +- Se si dispone di `bitsandbytes<0.37.0`, assicurarsi di eseguire su GPU NVIDIA che supportano tensor cores a 8 bit (Turing, Ampere o architetture più recenti - ad esempio T4, RTX20s RTX30s, A40-A100). Per `bitsandbytes>=0.37.0`, tutte le GPU dovrebbero essere supportate. +- Installare la versione corretta di `bitsandbytes` eseguendo: +`pip install bitsandbytes>=0.31.5`. +- Installare `accelerate` +`pip install accelerate>=0.12.0` + +### Esecuzione di modelli mixed-Int8 - configurazione per singola GPU + +Dopo aver installato le librerie necessarie, per caricare il tuo modello mixed 8-bit è il seguente: + +```py +from transformers import AutoModelForCausalLM + +model_name = "bigscience/bloom-2b5" +model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True) +``` + +Per la generazione di testo, si consiglia di: + +* utilizzare il metodo `generate()` del modello invece della funzione `pipeline()`. Sebbene l'inferenza sia possibile con la funzione `pipeline()`, essa non è ottimizzata per i modelli mixed-8bit e sarà più lenta rispetto all'uso del metodo `generate()`. Inoltre, alcune strategie di campionamento, come il campionamento nucleaus, non sono supportate dalla funzione `pipeline()` per i modelli mixed-8bit. +* collocare tutti gli ingressi sullo stesso dispositivo del modello. + +Ecco un semplice esempio: + +```py +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_name = "bigscience/bloom-2b5" +tokenizer = AutoTokenizer.from_pretrained(model_name) +model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True) + +text = "Hello, my llama is cute" +inputs = tokenizer(prompt, return_tensors="pt").to("cuda") +generated_ids = model.generate(**inputs) +outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True) +``` + + +### Esecuzione di modelli mixed-8bit - configurazione multi GPU + +Usare il seguente modo caricare il modello mixed-8bit su più GPU (stesso comando della configurazione a GPU singola): +```py +model_name = "bigscience/bloom-2b5" +model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True) +``` +Puoi controllare la RAM della GPU che si vuole allocare su ogni GPU usando `accelerate`. Utilizzare l'argomento `max_memory` come segue: + +```py +max_memory_mapping = {0: "1GB", 1: "2GB"} +model_name = "bigscience/bloom-3b" +model_8bit = AutoModelForCausalLM.from_pretrained( + model_name, device_map="auto", load_in_8bit=True, max_memory=max_memory_mapping +) +``` +In questo esempio, la prima GPU utilizzerà 1 GB di memoria e la seconda 2 GB. + +### Colab demos + +Con questo metodo è possibile inferire modelli che prima non era possibile inferire su Google Colab. +Guardate la demo per l'esecuzione di T5-11b (42GB in fp32)! Utilizzo la quantizzazione a 8 bit su Google Colab: + +[![Open In Colab: T5-11b demo](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing) + +Oppure questa demo di BLOOM-3B: + +[![Open In Colab: BLOOM-3b demo](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing) \ No newline at end of file From 0919d622912002ea29f1a80582aff2e667d84c90 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Tue, 21 Mar 2023 19:22:01 +0100 Subject: [PATCH 149/277] Time to Say Goodbye, torch 1.7 and 1.8 (#22291) * time to say goodbye, torch 1.7 and 1.8 * clean up torch_int_div * clean up is_torch_less_than_1_8-9 * update --------- Co-authored-by: ydshieh --- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- src/transformers/file_utils.py | 1 - src/transformers/generation/utils.py | 11 ++++++----- src/transformers/modeling_utils.py | 5 ----- .../models/big_bird/modeling_big_bird.py | 4 ++-- .../modeling_bigbird_pegasus.py | 3 +-- .../image_processing_conditional_detr.py | 6 ++---- .../modeling_conditional_detr.py | 5 ++--- .../data2vec/modeling_data2vec_audio.py | 3 +-- .../image_processing_deformable_detr.py | 6 ++---- .../modeling_deformable_detr.py | 6 +++--- .../models/deta/image_processing_deta.py | 3 +-- src/transformers/models/deta/modeling_deta.py | 6 +++--- src/transformers/models/detr/modeling_detr.py | 3 +-- .../models/hubert/modeling_hubert.py | 3 +-- .../models/layoutlmv2/modeling_layoutlmv2.py | 8 +++++--- .../image_processing_mask2former.py | 4 +--- .../maskformer/image_processing_maskformer.py | 4 +--- .../models/mctct/modeling_mctct.py | 7 ------- .../oneformer/image_processing_oneformer.py | 4 +--- src/transformers/models/sew/modeling_sew.py | 3 +-- .../models/sew_d/modeling_sew_d.py | 4 ++-- .../models/speecht5/modeling_speecht5.py | 3 +-- .../modeling_table_transformer.py | 3 +-- .../models/tapas/modeling_tapas.py | 3 +-- .../models/unispeech/modeling_unispeech.py | 3 +-- .../unispeech_sat/modeling_unispeech_sat.py | 3 +-- .../models/wav2vec2/modeling_wav2vec2.py | 3 +-- .../modeling_wav2vec2_conformer.py | 3 +-- .../models/wavlm/modeling_wavlm.py | 3 +-- src/transformers/onnx/convert.py | 4 ---- src/transformers/pytorch_utils.py | 12 ------------ src/transformers/utils/__init__.py | 1 - src/transformers/utils/import_utils.py | 9 +-------- tests/generation/test_beam_search.py | 9 ++++++--- tests/models/bloom/test_modeling_bloom.py | 6 +----- tests/models/mctct/test_modeling_mctct.py | 4 ---- tests/models/swin/test_modeling_swin.py | 5 +---- .../test_modeling_trajectory_transformer.py | 4 ---- .../models/wav2vec2/test_modeling_wav2vec2.py | 19 ++++++------------- tests/onnx/test_onnx_v2.py | 10 ---------- 42 files changed, 61 insertions(+), 149 deletions(-) diff --git a/setup.py b/setup.py index c28387a3d454..70197c209f0f 100644 --- a/setup.py +++ b/setup.py @@ -171,7 +171,7 @@ "timeout-decorator", "timm", "tokenizers>=0.11.1,!=0.11.3,<0.14", - "torch>=1.7,!=1.12.0", + "torch>=1.9,!=1.12.0", "torchaudio", "torchvision", "pyctcdecode>=0.4.0", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index aa638a6a9f60..aa23c1ad06cc 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -77,7 +77,7 @@ "timeout-decorator": "timeout-decorator", "timm": "timm", "tokenizers": "tokenizers>=0.11.1,!=0.11.3,<0.14", - "torch": "torch>=1.7,!=1.12.0", + "torch": "torch>=1.9,!=1.12.0", "torchaudio": "torchaudio", "torchvision": "torchvision", "pyctcdecode": "pyctcdecode>=0.4.0", diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 263ebc94e345..da24760118c6 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -115,7 +115,6 @@ is_torch_cuda_available, is_torch_fx_available, is_torch_fx_proxy, - is_torch_onnx_dict_inputs_support_available, is_torch_tf32_available, is_torch_tpu_available, is_torchaudio_available, diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index b0b4e577d3c3..eb0e0ed4c21f 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -32,7 +32,6 @@ MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING, MODEL_FOR_VISION_2_SEQ_MAPPING, ) -from ..pytorch_utils import torch_int_div from ..utils import ModelOutput, logging from .beam_constraints import DisjunctiveConstraint, PhrasalConstraint from .beam_search import BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer @@ -2795,7 +2794,7 @@ def beam_search( next_token_scores, 2 * num_beams, dim=1, largest=True, sorted=True ) - next_indices = torch_int_div(next_tokens, vocab_size) + next_indices = torch.div(next_tokens, vocab_size, rounding_mode="floor") next_tokens = next_tokens % vocab_size # stateless @@ -3129,7 +3128,7 @@ def beam_sample( next_token_scores, _indices = torch.sort(next_token_scores, descending=True, dim=1) next_tokens = torch.gather(next_tokens, -1, _indices) - next_indices = torch_int_div(next_tokens, vocab_size) + next_indices = torch.div(next_tokens, vocab_size, rounding_mode="floor") next_tokens = next_tokens % vocab_size # stateless @@ -3473,7 +3472,7 @@ def group_beam_search( next_token_scores, 2 * group_size, dim=1, largest=True, sorted=True ) - next_indices = torch_int_div(next_tokens, vocab_size) + next_indices = torch.div(next_tokens, vocab_size, rounding_mode="floor") next_tokens = next_tokens % vocab_size # stateless @@ -3503,7 +3502,9 @@ def group_beam_search( # (beam_idx // group_size) -> batch_idx # (beam_idx % group_size) -> offset of idx inside the group reordering_indices[batch_group_indices] = ( - num_beams * torch_int_div(beam_idx, group_size) + group_start_idx + (beam_idx % group_size) + num_beams * torch.div(beam_idx, group_size, rounding_mode="floor") + + group_start_idx + + (beam_idx % group_size) ) # Store scores, attentions and hidden_states when required diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 901677a604dd..94bbe4a9d5f9 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -539,9 +539,6 @@ def _move_model_to_meta(model, loaded_state_dict_keys, start_prefix): """ - # meta device was added in pt=1.9 - require_version_core("torch>=1.9") - # dematerialize param storage for keys that are going to be replaced by state_dict, by # putting those on the meta device for k in loaded_state_dict_keys: @@ -2100,8 +2097,6 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P raise ValueError("Passing along a `device_map` requires `low_cpu_mem_usage=True`") if low_cpu_mem_usage: - # low_cpu_mem_usage requires PyTorch >= 1.9 to have the meta device. - require_version_core("torch>=1.9") if device_map is not None: # The max memory utils require PyTorch >= 1.10 to have torch.cuda.mem_get_info. require_version_core("torch>=1.10") diff --git a/src/transformers/models/big_bird/modeling_big_bird.py b/src/transformers/models/big_bird/modeling_big_bird.py index ba3b82555ff7..4f90deb506b7 100755 --- a/src/transformers/models/big_bird/modeling_big_bird.py +++ b/src/transformers/models/big_bird/modeling_big_bird.py @@ -37,7 +37,7 @@ TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import apply_chunking_to_forward, torch_int_div +from ...pytorch_utils import apply_chunking_to_forward from ...utils import ( ModelOutput, add_code_sample_docstrings, @@ -972,7 +972,7 @@ def torch_gather_b2(params, indices): num_indices_to_pick_from = params.shape[2] shift = torch.arange(indices.shape[0] * indices.shape[1] * num_indices_to_gather, device=indices.device) - indices_shift = torch_int_div(shift, num_indices_to_gather) * num_indices_to_pick_from + indices_shift = torch.div(shift, num_indices_to_gather, rounding_mode="floor") * num_indices_to_pick_from flattened_indices = indices.view(-1) + indices_shift flattened_params = params.reshape(-1, params.shape[-2], params.shape[-1]) diff --git a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py index 1def588a514f..ad22199e66e6 100755 --- a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py +++ b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py @@ -36,7 +36,6 @@ Seq2SeqSequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( add_code_sample_docstrings, add_end_docstrings, @@ -791,7 +790,7 @@ def torch_gather_b2(params, indices): num_indices_to_pick_from = params.shape[2] shift = torch.arange(indices.shape[0] * indices.shape[1] * num_indices_to_gather, device=indices.device) - indices_shift = torch_int_div(shift, num_indices_to_gather) * num_indices_to_pick_from + indices_shift = torch.div(shift, num_indices_to_gather, rounding_mode="floor") * num_indices_to_pick_from flattened_indices = indices.view(-1) + indices_shift flattened_params = params.reshape(-1, params.shape[-2], params.shape[-1]) diff --git a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py index a496f7787fb3..8a146ccea0d8 100644 --- a/src/transformers/models/conditional_detr/image_processing_conditional_detr.py +++ b/src/transformers/models/conditional_detr/image_processing_conditional_detr.py @@ -68,8 +68,6 @@ import torch from torch import nn - from transformers.pytorch_utils import torch_int_div - if is_vision_available(): import PIL @@ -1314,7 +1312,7 @@ def post_process(self, outputs, target_sizes): prob = out_logits.sigmoid() topk_values, topk_indexes = torch.topk(prob.view(out_logits.shape[0], -1), 300, dim=1) scores = topk_values - topk_boxes = torch_int_div(topk_indexes, out_logits.shape[2]) + topk_boxes = torch.div(topk_indexes, out_logits.shape[2], rounding_mode="floor") labels = topk_indexes % out_logits.shape[2] boxes = center_to_corners_format(out_bbox) boxes = torch.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) @@ -1360,7 +1358,7 @@ def post_process_object_detection( prob = out_logits.sigmoid() topk_values, topk_indexes = torch.topk(prob.view(out_logits.shape[0], -1), 100, dim=1) scores = topk_values - topk_boxes = torch_int_div(topk_indexes, out_logits.shape[2]) + topk_boxes = torch.div(topk_indexes, out_logits.shape[2], rounding_mode="floor") labels = topk_indexes % out_logits.shape[2] boxes = center_to_corners_format(out_bbox) boxes = torch.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) diff --git a/src/transformers/models/conditional_detr/modeling_conditional_detr.py b/src/transformers/models/conditional_detr/modeling_conditional_detr.py index eb67b1e25bda..6110fb4491ce 100644 --- a/src/transformers/models/conditional_detr/modeling_conditional_detr.py +++ b/src/transformers/models/conditional_detr/modeling_conditional_detr.py @@ -26,7 +26,6 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttentions, Seq2SeqModelOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( ModelOutput, add_start_docstrings, @@ -452,7 +451,7 @@ def forward(self, pixel_values, pixel_mask): x_embed = x_embed / (x_embed[:, :, -1:] + 1e-6) * self.scale dim_t = torch.arange(self.embedding_dim, dtype=torch.float32, device=pixel_values.device) - dim_t = self.temperature ** (2 * torch_int_div(dim_t, 2) / self.embedding_dim) + dim_t = self.temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / self.embedding_dim) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t @@ -504,7 +503,7 @@ def build_position_encoding(config): def gen_sine_position_embeddings(pos_tensor): scale = 2 * math.pi dim_t = torch.arange(128, dtype=torch.float32, device=pos_tensor.device) - dim_t = 10000 ** (2 * torch_int_div(dim_t, 2) / 128) + dim_t = 10000 ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / 128) x_embed = pos_tensor[:, :, 0] * scale y_embed = pos_tensor[:, :, 1] * scale pos_x = x_embed[:, :, None] / dim_t diff --git a/src/transformers/models/data2vec/modeling_data2vec_audio.py b/src/transformers/models/data2vec/modeling_data2vec_audio.py index e659d3078199..168f342acd32 100755 --- a/src/transformers/models/data2vec/modeling_data2vec_audio.py +++ b/src/transformers/models/data2vec/modeling_data2vec_audio.py @@ -35,7 +35,6 @@ XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_data2vec_audio import Data2VecAudioConfig @@ -731,7 +730,7 @@ def _get_feat_extract_output_lengths( def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py index 01ff54471904..07cafe149e15 100644 --- a/src/transformers/models/deformable_detr/image_processing_deformable_detr.py +++ b/src/transformers/models/deformable_detr/image_processing_deformable_detr.py @@ -68,8 +68,6 @@ import torch from torch import nn - from ...pytorch_utils import torch_int_div - if is_vision_available(): import PIL @@ -1312,7 +1310,7 @@ def post_process(self, outputs, target_sizes): prob = out_logits.sigmoid() topk_values, topk_indexes = torch.topk(prob.view(out_logits.shape[0], -1), 100, dim=1) scores = topk_values - topk_boxes = torch_int_div(topk_indexes, out_logits.shape[2]) + topk_boxes = torch.div(topk_indexes, out_logits.shape[2], rounding_mode="floor") labels = topk_indexes % out_logits.shape[2] boxes = center_to_corners_format(out_bbox) boxes = torch.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) @@ -1357,7 +1355,7 @@ def post_process_object_detection( prob = out_logits.sigmoid() topk_values, topk_indexes = torch.topk(prob.view(out_logits.shape[0], -1), 100, dim=1) scores = topk_values - topk_boxes = torch_int_div(topk_indexes, out_logits.shape[2]) + topk_boxes = torch.div(topk_indexes, out_logits.shape[2], rounding_mode="floor") labels = topk_indexes % out_logits.shape[2] boxes = center_to_corners_format(out_bbox) boxes = torch.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) diff --git a/src/transformers/models/deformable_detr/modeling_deformable_detr.py b/src/transformers/models/deformable_detr/modeling_deformable_detr.py index edb1b8349e86..bc6959ddcc39 100755 --- a/src/transformers/models/deformable_detr/modeling_deformable_detr.py +++ b/src/transformers/models/deformable_detr/modeling_deformable_detr.py @@ -41,7 +41,7 @@ ) from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import meshgrid, torch_int_div +from ...pytorch_utils import meshgrid from ...utils import is_ninja_available, logging from ..auto import AutoBackbone from .configuration_deformable_detr import DeformableDetrConfig @@ -497,7 +497,7 @@ def forward(self, pixel_values, pixel_mask): x_embed = (x_embed - 0.5) / (x_embed[:, :, -1:] + eps) * self.scale dim_t = torch.arange(self.embedding_dim, dtype=torch.float32, device=pixel_values.device) - dim_t = self.temperature ** (2 * torch_int_div(dim_t, 2) / self.embedding_dim) + dim_t = self.temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / self.embedding_dim) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t @@ -1552,7 +1552,7 @@ def get_proposal_pos_embed(self, proposals): scale = 2 * math.pi dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=proposals.device) - dim_t = temperature ** (2 * torch_int_div(dim_t, 2) / num_pos_feats) + dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) # batch_size, num_queries, 4 proposals = proposals.sigmoid() * scale # batch_size, num_queries, 4, 128 diff --git a/src/transformers/models/deta/image_processing_deta.py b/src/transformers/models/deta/image_processing_deta.py index 8afc69fd31b4..eda4fdff167d 100644 --- a/src/transformers/models/deta/image_processing_deta.py +++ b/src/transformers/models/deta/image_processing_deta.py @@ -63,7 +63,6 @@ if is_torch_available(): import torch - from ...pytorch_utils import torch_int_div if is_torchvision_available(): from torchvision.ops.boxes import batched_nms @@ -967,7 +966,7 @@ def post_process_object_detection( all_scores = prob.view(batch_size, num_queries * num_labels).to(out_logits.device) all_indexes = torch.arange(num_queries * num_labels)[None].repeat(batch_size, 1).to(out_logits.device) - all_boxes = torch_int_div(all_indexes, out_logits.shape[2]) + all_boxes = torch.div(all_indexes, out_logits.shape[2], rounding_mode="floor") all_labels = all_indexes % out_logits.shape[2] boxes = center_to_corners_format(out_bbox) diff --git a/src/transformers/models/deta/modeling_deta.py b/src/transformers/models/deta/modeling_deta.py index 33706069c499..6fd2e8fdd184 100644 --- a/src/transformers/models/deta/modeling_deta.py +++ b/src/transformers/models/deta/modeling_deta.py @@ -36,7 +36,7 @@ ) from ...modeling_outputs import BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import meshgrid, torch_int_div +from ...pytorch_utils import meshgrid from ...utils import is_torchvision_available, logging, requires_backends from ..auto import AutoBackbone from .configuration_deta import DetaConfig @@ -399,7 +399,7 @@ def forward(self, pixel_values, pixel_mask): x_embed = (x_embed - 0.5) / (x_embed[:, :, -1:] + eps) * self.scale dim_t = torch.arange(self.embedding_dim, dtype=torch.float32, device=pixel_values.device) - dim_t = self.temperature ** (2 * torch_int_div(dim_t, 2) / self.embedding_dim) + dim_t = self.temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / self.embedding_dim) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t @@ -1463,7 +1463,7 @@ def get_proposal_pos_embed(self, proposals): scale = 2 * math.pi dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=proposals.device) - dim_t = temperature ** (2 * torch_int_div(dim_t, 2) / num_pos_feats) + dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) # batch_size, num_queries, 4 proposals = proposals.sigmoid() * scale # batch_size, num_queries, 4, 128 diff --git a/src/transformers/models/detr/modeling_detr.py b/src/transformers/models/detr/modeling_detr.py index 814a09c37b9f..bd4eae128394 100644 --- a/src/transformers/models/detr/modeling_detr.py +++ b/src/transformers/models/detr/modeling_detr.py @@ -26,7 +26,6 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttentions, Seq2SeqModelOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( ModelOutput, add_start_docstrings, @@ -442,7 +441,7 @@ def forward(self, pixel_values, pixel_mask): x_embed = x_embed / (x_embed[:, :, -1:] + 1e-6) * self.scale dim_t = torch.arange(self.embedding_dim, dtype=torch.float32, device=pixel_values.device) - dim_t = self.temperature ** (2 * torch_int_div(dim_t, 2) / self.embedding_dim) + dim_t = self.temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / self.embedding_dim) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t diff --git a/src/transformers/models/hubert/modeling_hubert.py b/src/transformers/models/hubert/modeling_hubert.py index 8c7a1a15162f..1c2493825642 100755 --- a/src/transformers/models/hubert/modeling_hubert.py +++ b/src/transformers/models/hubert/modeling_hubert.py @@ -27,7 +27,6 @@ from ...deepspeed import is_deepspeed_zero3_enabled from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( add_code_sample_docstrings, add_start_docstrings, @@ -871,7 +870,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py index 9e9c7033f18a..5a6f39ce31a6 100755 --- a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py +++ b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py @@ -31,7 +31,7 @@ TokenClassifierOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import apply_chunking_to_forward, torch_int_div +from ...pytorch_utils import apply_chunking_to_forward from ...utils import ( add_start_docstrings, add_start_docstrings_to_model_forward, @@ -770,7 +770,7 @@ def _calc_img_embeddings(self, image, bbox, position_ids): return embeddings def _calc_visual_bbox(self, image_feature_pool_shape, bbox, device, final_shape): - visual_bbox_x = torch_int_div( + visual_bbox_x = torch.div( torch.arange( 0, 1000 * (image_feature_pool_shape[1] + 1), @@ -779,8 +779,9 @@ def _calc_visual_bbox(self, image_feature_pool_shape, bbox, device, final_shape) dtype=bbox.dtype, ), self.config.image_feature_pool_shape[1], + rounding_mode="floor", ) - visual_bbox_y = torch_int_div( + visual_bbox_y = torch.div( torch.arange( 0, 1000 * (self.config.image_feature_pool_shape[0] + 1), @@ -789,6 +790,7 @@ def _calc_visual_bbox(self, image_feature_pool_shape, bbox, device, final_shape) dtype=bbox.dtype, ), self.config.image_feature_pool_shape[0], + rounding_mode="floor", ) visual_bbox = torch.stack( [ diff --git a/src/transformers/models/mask2former/image_processing_mask2former.py b/src/transformers/models/mask2former/image_processing_mask2former.py index 07642faf24bf..234e784bd67f 100644 --- a/src/transformers/models/mask2former/image_processing_mask2former.py +++ b/src/transformers/models/mask2former/image_processing_mask2former.py @@ -57,8 +57,6 @@ import torch from torch import nn - from ...pytorch_utils import torch_int_div - # Copied from transformers.models.detr.image_processing_detr.max_across_indices def max_across_indices(values: Iterable[Any]) -> List[Any]: @@ -1009,7 +1007,7 @@ def post_process_instance_segmentation( scores_per_image, topk_indices = scores.flatten(0, 1).topk(num_queries, sorted=False) labels_per_image = labels[topk_indices] - topk_indices = torch_int_div(topk_indices, num_classes) + topk_indices = torch.div(topk_indices, num_classes, rounding_mode="floor") mask_pred = mask_pred[topk_indices] pred_masks = (mask_pred > 0).float() diff --git a/src/transformers/models/maskformer/image_processing_maskformer.py b/src/transformers/models/maskformer/image_processing_maskformer.py index ef4314869b4d..bfdc07431e35 100644 --- a/src/transformers/models/maskformer/image_processing_maskformer.py +++ b/src/transformers/models/maskformer/image_processing_maskformer.py @@ -61,8 +61,6 @@ import torch from torch import nn - from ...pytorch_utils import torch_int_div - # Copied from transformers.models.detr.image_processing_detr.max_across_indices def max_across_indices(values: Iterable[Any]) -> List[Any]: @@ -1077,7 +1075,7 @@ def post_process_instance_segmentation( scores_per_image, topk_indices = scores.flatten(0, 1).topk(num_queries, sorted=False) labels_per_image = labels[topk_indices] - topk_indices = torch_int_div(topk_indices, num_classes) + topk_indices = torch.div(topk_indices, num_classes, rounding_mode="floor") mask_pred = mask_pred[topk_indices] pred_masks = (mask_pred > 0).float() diff --git a/src/transformers/models/mctct/modeling_mctct.py b/src/transformers/models/mctct/modeling_mctct.py index 3effb52de533..08e280b3ccf9 100755 --- a/src/transformers/models/mctct/modeling_mctct.py +++ b/src/transformers/models/mctct/modeling_mctct.py @@ -33,19 +33,12 @@ find_pruneable_heads_and_indices, prune_linear_layer, ) -from ...pytorch_utils import is_torch_less_than_1_9 from ...utils import logging from .configuration_mctct import MCTCTConfig logger = logging.get_logger(__name__) -if is_torch_less_than_1_9: - logger.warning( - f"You are using torch=={torch.__version__}, but torch>=1.9.0 is required to use MCTCTModel. Please upgrade" - " torch." - ) - _HIDDEN_STATES_START_POSITION = 1 _CONFIG_FOR_DOC = "MCTCTConfig" diff --git a/src/transformers/models/oneformer/image_processing_oneformer.py b/src/transformers/models/oneformer/image_processing_oneformer.py index 67eadd4e841a..a99a7182b858 100644 --- a/src/transformers/models/oneformer/image_processing_oneformer.py +++ b/src/transformers/models/oneformer/image_processing_oneformer.py @@ -58,8 +58,6 @@ import torch from torch import nn - from ...pytorch_utils import torch_int_div - # Copied from transformers.models.detr.image_processing_detr.max_across_indices def max_across_indices(values: Iterable[Any]) -> List[Any]: @@ -1122,7 +1120,7 @@ def post_process_instance_segmentation( scores_per_image, topk_indices = scores.flatten(0, 1).topk(num_queries, sorted=False) labels_per_image = labels[topk_indices] - topk_indices = torch_int_div(topk_indices, num_classes) + topk_indices = torch.div(topk_indices, num_classes, rounding_mode="floor") # mask_pred = mask_pred.unsqueeze(1).repeat(1, self.sem_seg_head.num_classes, 1).flatten(0, 1) mask_pred = masks_queries_logits[i][topk_indices] diff --git a/src/transformers/models/sew/modeling_sew.py b/src/transformers/models/sew/modeling_sew.py index 836c38f8f4bc..3e41496ea13b 100644 --- a/src/transformers/models/sew/modeling_sew.py +++ b/src/transformers/models/sew/modeling_sew.py @@ -28,7 +28,6 @@ from ...deepspeed import is_deepspeed_zero3_enabled from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_sew import SEWConfig @@ -770,7 +769,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/sew_d/modeling_sew_d.py b/src/transformers/models/sew_d/modeling_sew_d.py index 02a8477d146a..7cdce062eee3 100644 --- a/src/transformers/models/sew_d/modeling_sew_d.py +++ b/src/transformers/models/sew_d/modeling_sew_d.py @@ -29,7 +29,7 @@ from ...deepspeed import is_deepspeed_zero3_enabled from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import softmax_backward_data, torch_int_div +from ...pytorch_utils import softmax_backward_data from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_sew_d import SEWDConfig @@ -1305,7 +1305,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/speecht5/modeling_speecht5.py b/src/transformers/models/speecht5/modeling_speecht5.py index 975f483395be..01ac79af472e 100644 --- a/src/transformers/models/speecht5/modeling_speecht5.py +++ b/src/transformers/models/speecht5/modeling_speecht5.py @@ -34,7 +34,6 @@ Seq2SeqSpectrogramOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings from .configuration_speecht5 import SpeechT5Config, SpeechT5HifiGanConfig @@ -620,7 +619,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/table_transformer/modeling_table_transformer.py b/src/transformers/models/table_transformer/modeling_table_transformer.py index ff4bca7a5a99..6716a78be3b5 100644 --- a/src/transformers/models/table_transformer/modeling_table_transformer.py +++ b/src/transformers/models/table_transformer/modeling_table_transformer.py @@ -26,7 +26,6 @@ from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithCrossAttentions, Seq2SeqModelOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( ModelOutput, add_start_docstrings, @@ -380,7 +379,7 @@ def forward(self, pixel_values, pixel_mask): x_embed = x_embed / (x_embed[:, :, -1:] + 1e-6) * self.scale dim_t = torch.arange(self.embedding_dim, dtype=torch.float32, device=pixel_values.device) - dim_t = self.temperature ** (2 * torch_int_div(dim_t, 2) / self.embedding_dim) + dim_t = self.temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / self.embedding_dim) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py index 2bb4e2baae91..c215f0e0d3a5 100644 --- a/src/transformers/models/tapas/modeling_tapas.py +++ b/src/transformers/models/tapas/modeling_tapas.py @@ -33,7 +33,6 @@ apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer, - torch_int_div, ) from ...utils import ( ModelOutput, @@ -1637,7 +1636,7 @@ def __init__(self, outer_index, inner_index): def project_outer(self, index): """Projects an index with the same index set onto the outer components.""" - indices = torch_int_div(index.indices, self.inner_index.num_segments).type(torch.long) + indices = torch.div(index.indices, self.inner_index.num_segments, rounding_mode="floor").type(torch.long) return IndexMap(indices=indices, num_segments=self.outer_index.num_segments, batch_dims=index.batch_dims) def project_inner(self, index): diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py index c55a90bb6a35..71dcaad1193d 100755 --- a/src/transformers/models/unispeech/modeling_unispeech.py +++ b/src/transformers/models/unispeech/modeling_unispeech.py @@ -29,7 +29,6 @@ from ...deepspeed import is_deepspeed_zero3_enabled from ...modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput, Wav2Vec2BaseModelOutput from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( ModelOutput, add_code_sample_docstrings, @@ -981,7 +980,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py index baf0f93a090f..ce3f5b80ca13 100755 --- a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py +++ b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py @@ -36,7 +36,6 @@ XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( ModelOutput, add_code_sample_docstrings, @@ -995,7 +994,7 @@ def _get_feat_extract_output_lengths(self, input_lengths: Union[torch.LongTensor def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/wav2vec2/modeling_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_wav2vec2.py index 856ce5656087..bcf7b7a3ad8d 100755 --- a/src/transformers/models/wav2vec2/modeling_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_wav2vec2.py @@ -37,7 +37,6 @@ XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( ModelOutput, add_code_sample_docstrings, @@ -1098,7 +1097,7 @@ def _get_feat_extract_output_lengths( def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py b/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py index 0dfac20c06da..94ce66e4f3d7 100644 --- a/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py +++ b/src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py @@ -35,7 +35,6 @@ XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import ( ModelOutput, add_code_sample_docstrings, @@ -1143,7 +1142,7 @@ def _get_feat_extract_output_lengths( def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/models/wavlm/modeling_wavlm.py b/src/transformers/models/wavlm/modeling_wavlm.py index 6347aafab720..718c4d61dd59 100755 --- a/src/transformers/models/wavlm/modeling_wavlm.py +++ b/src/transformers/models/wavlm/modeling_wavlm.py @@ -36,7 +36,6 @@ XVectorOutput, ) from ...modeling_utils import PreTrainedModel -from ...pytorch_utils import torch_int_div from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_wavlm import WavLMConfig @@ -1019,7 +1018,7 @@ def _get_feat_extract_output_lengths( def _conv_out_length(input_length, kernel_size, stride): # 1D convolutional layer output length formula taken # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html - return torch_int_div(input_length - kernel_size, stride) + 1 + return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1 for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride): input_lengths = _conv_out_length(input_lengths, kernel_size, stride) diff --git a/src/transformers/onnx/convert.py b/src/transformers/onnx/convert.py index 918134d31129..288c2574e190 100644 --- a/src/transformers/onnx/convert.py +++ b/src/transformers/onnx/convert.py @@ -26,7 +26,6 @@ TensorType, is_tf_available, is_torch_available, - is_torch_onnx_dict_inputs_support_available, logging, ) from .config import OnnxConfig @@ -339,9 +338,6 @@ def export( if is_torch_available(): from ..utils import torch_version - if not is_torch_onnx_dict_inputs_support_available(): - raise AssertionError(f"Unsupported PyTorch version, minimum required is 1.8.0, got: {torch_version}") - if not config.is_torch_support_available: logger.warning( f"Unsupported PyTorch version for this model. Minimum required is {config.torch_onnx_minimum_version}," diff --git a/src/transformers/pytorch_utils.py b/src/transformers/pytorch_utils.py index 15549f2233e4..4e477afd9e57 100644 --- a/src/transformers/pytorch_utils.py +++ b/src/transformers/pytorch_utils.py @@ -27,22 +27,10 @@ parsed_torch_version_base = version.parse(version.parse(torch.__version__).base_version) -is_torch_less_than_1_8 = parsed_torch_version_base < version.parse("1.8.0") -is_torch_less_than_1_9 = parsed_torch_version_base < version.parse("1.9.0") is_torch_greater_or_equal_than_1_10 = parsed_torch_version_base >= version.parse("1.10") is_torch_less_than_1_11 = parsed_torch_version_base < version.parse("1.11") -def torch_int_div(tensor1, tensor2): - """ - A function that performs integer division across different versions of PyTorch. - """ - if is_torch_less_than_1_8: - return tensor1 // tensor2 - else: - return torch.div(tensor1, tensor2, rounding_mode="floor") - - def softmax_backward_data(parent, grad_output, output, dim, self): """ A function that calls the internal `_softmax_backward_data` PyTorch method and that adjusts the arguments according diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index 26371782f87d..b9b1771e6831 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -151,7 +151,6 @@ is_torch_fx_available, is_torch_fx_proxy, is_torch_neuroncore_available, - is_torch_onnx_dict_inputs_support_available, is_torch_tensorrt_fx_available, is_torch_tf32_available, is_torch_tpu_available, diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index cfa7f44d9849..ceb876040fcd 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -276,7 +276,6 @@ # This is the version of torch required to run torch.fx features and torch.onnx with dictionary inputs. TORCH_FX_REQUIRED_VERSION = version.parse("1.10") -TORCH_ONNX_DICT_INPUTS_MINIMUM_VERSION = version.parse("1.8") def is_kenlm_available(): @@ -388,7 +387,7 @@ def is_torch_tf32_available(): torch_version = None -_torch_fx_available = _torch_onnx_dict_inputs_support_available = False +_torch_fx_available = False if _torch_available: torch_version = version.parse(importlib_metadata.version("torch")) _torch_fx_available = (torch_version.major, torch_version.minor) >= ( @@ -396,8 +395,6 @@ def is_torch_tf32_available(): TORCH_FX_REQUIRED_VERSION.minor, ) - _torch_onnx_dict_inputs_support_available = torch_version >= TORCH_ONNX_DICT_INPUTS_MINIMUM_VERSION - def is_torch_fx_available(): return _torch_fx_available @@ -407,10 +404,6 @@ def is_bs4_available(): return importlib.util.find_spec("bs4") is not None -def is_torch_onnx_dict_inputs_support_available(): - return _torch_onnx_dict_inputs_support_available - - def is_tf_available(): return _tf_available diff --git a/tests/generation/test_beam_search.py b/tests/generation/test_beam_search.py index e35e8d9b81c9..47d3b4b38a7b 100644 --- a/tests/generation/test_beam_search.py +++ b/tests/generation/test_beam_search.py @@ -32,7 +32,6 @@ DisjunctiveConstraint, PhrasalConstraint, ) - from transformers.pytorch_utils import torch_int_div class BeamSearchTester: @@ -161,7 +160,9 @@ def cut_expected_tensor(tensor): expected_output_scores = cut_expected_tensor(next_scores) # add num_beams * batch_idx - offset = torch_int_div(torch.arange(self.num_beams * self.batch_size, device=torch_device), self.num_beams) + offset = torch.div( + torch.arange(self.num_beams * self.batch_size, device=torch_device), self.num_beams, rounding_mode="floor" + ) expected_output_indices = cut_expected_tensor(next_indices) + offset * self.num_beams self.parent.assertListEqual(expected_output_tokens.tolist(), output_tokens.tolist()) @@ -398,7 +399,9 @@ def cut_expected_tensor(tensor): expected_output_scores = cut_expected_tensor(next_scores) # add num_beams * batch_idx - offset = torch_int_div(torch.arange(self.num_beams * self.batch_size, device=torch_device), self.num_beams) + offset = torch.div( + torch.arange(self.num_beams * self.batch_size, device=torch_device), self.num_beams, rounding_mode="floor" + ) expected_output_indices = cut_expected_tensor(next_indices) + offset * self.num_beams self.parent.assertListEqual(expected_output_tokens.tolist(), output_tokens.tolist()) diff --git a/tests/models/bloom/test_modeling_bloom.py b/tests/models/bloom/test_modeling_bloom.py index 9dc803a3ba43..617998cc61b3 100644 --- a/tests/models/bloom/test_modeling_bloom.py +++ b/tests/models/bloom/test_modeling_bloom.py @@ -38,10 +38,9 @@ BloomModel, BloomTokenizerFast, ) - from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_10, is_torch_less_than_1_9 + from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_10 else: is_torch_greater_or_equal_than_1_10 = False - is_torch_less_than_1_9 = True @require_torch @@ -751,9 +750,6 @@ def test_embeddings(self): self.assertAlmostEqual(EMBEDDINGS_DS_AFTER_LN[key][idx], output_dict_norm[key][idx], places=1) @require_torch - @unittest.skipIf( - is_torch_less_than_1_9, reason="Test failed with torch < 1.9 (`min_cuda` not implemented for `BFloat16`)" - ) def test_hidden_states_transformers(self): cuda_available = torch.cuda.is_available() model = BloomModel.from_pretrained(self.path_bigscience_model, use_cache=False, torch_dtype="auto").to( diff --git a/tests/models/mctct/test_modeling_mctct.py b/tests/models/mctct/test_modeling_mctct.py index c488c3e75d83..7fa9d99a3fa0 100644 --- a/tests/models/mctct/test_modeling_mctct.py +++ b/tests/models/mctct/test_modeling_mctct.py @@ -32,9 +32,6 @@ import torch from transformers import MCTCTForCTC, MCTCTModel, MCTCTProcessor - from transformers.pytorch_utils import is_torch_less_than_1_9 -else: - is_torch_less_than_1_9 = True class MCTCTModelTester: @@ -265,7 +262,6 @@ def prepare_config_and_inputs_for_common(self): @require_torch -@unittest.skipIf(is_torch_less_than_1_9, "MCTCT is only available in torch v1.9+") class MCTCTModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = (MCTCTForCTC, MCTCTModel) if is_torch_available() else () pipeline_model_mapping = ( diff --git a/tests/models/swin/test_modeling_swin.py b/tests/models/swin/test_modeling_swin.py index 804bffd31fa8..f519a0204f87 100644 --- a/tests/models/swin/test_modeling_swin.py +++ b/tests/models/swin/test_modeling_swin.py @@ -33,9 +33,7 @@ from transformers import SwinBackbone, SwinForImageClassification, SwinForMaskedImageModeling, SwinModel from transformers.models.swin.modeling_swin import SWIN_PRETRAINED_MODEL_ARCHIVE_LIST - from transformers.pytorch_utils import is_torch_less_than_1_9 -else: - is_torch_less_than_1_9 = True + if is_vision_available(): from PIL import Image @@ -266,7 +264,6 @@ def test_model(self): def test_multi_gpu_data_parallel_forward(self): pass - @unittest.skipIf(is_torch_less_than_1_9, reason="This test fails for SwinModel when torch < 1.9") def test_training_gradient_checkpointing(self): super().test_training_gradient_checkpointing() diff --git a/tests/models/trajectory_transformer/test_modeling_trajectory_transformer.py b/tests/models/trajectory_transformer/test_modeling_trajectory_transformer.py index 0b9be4918c98..cf553bdf75b3 100644 --- a/tests/models/trajectory_transformer/test_modeling_trajectory_transformer.py +++ b/tests/models/trajectory_transformer/test_modeling_trajectory_transformer.py @@ -36,9 +36,6 @@ from transformers.models.trajectory_transformer.modeling_trajectory_transformer import ( TRAJECTORY_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST, ) - from transformers.pytorch_utils import is_torch_less_than_1_9 -else: - is_torch_less_than_1_9 = True class TrajectoryTransformerModelTester: @@ -199,7 +196,6 @@ def test_training(self): ).loss loss.backward() - @unittest.skipIf(is_torch_less_than_1_9, reason="This test fails for TrajectoryTransformerModel when torch < 1.9") def test_training_gradient_checkpointing(self): if not self.model_tester.is_training: return diff --git a/tests/models/wav2vec2/test_modeling_wav2vec2.py b/tests/models/wav2vec2/test_modeling_wav2vec2.py index cd98d4103438..f3e93b670c32 100644 --- a/tests/models/wav2vec2/test_modeling_wav2vec2.py +++ b/tests/models/wav2vec2/test_modeling_wav2vec2.py @@ -71,9 +71,6 @@ _compute_mask_indices, _sample_negative_indices, ) - from transformers.pytorch_utils import is_torch_less_than_1_9, torch_int_div -else: - is_torch_less_than_1_9 = True if is_torchaudio_available(): @@ -1217,7 +1214,9 @@ def test_sample_negatives(self): sequence_length = 10 hidden_size = 4 num_negatives = 3 - sequence = torch_int_div(torch.arange(sequence_length * hidden_size, device=torch_device), hidden_size) + sequence = torch.div( + torch.arange(sequence_length * hidden_size, device=torch_device), hidden_size, rounding_mode="floor" + ) features = sequence.view(sequence_length, hidden_size) # each value in vector consits of same value features = features[None, :].expand(batch_size, sequence_length, hidden_size).contiguous() @@ -1245,7 +1244,9 @@ def test_sample_negatives_with_mask(self): mask = torch.ones((batch_size, sequence_length), dtype=torch.long, device=torch_device) mask[-1, sequence_length // 2 :] = 0 - sequence = torch_int_div(torch.arange(sequence_length * hidden_size, device=torch_device), hidden_size) + sequence = torch.div( + torch.arange(sequence_length * hidden_size, device=torch_device), hidden_size, rounding_mode="floor" + ) features = sequence.view(sequence_length, hidden_size) # each value in vector consits of same value features = features[None, :].expand(batch_size, sequence_length, hidden_size).contiguous() @@ -1651,10 +1652,6 @@ def test_phoneme_recognition(self): @require_pyctcdecode @require_torchaudio - @unittest.skipIf( - is_torch_less_than_1_9, - reason="`torchaudio.functional.resample` needs torchaudio >= 0.9 which requires torch >= 0.9", - ) def test_wav2vec2_with_lm(self): ds = load_dataset("common_voice", "es", split="test", streaming=True) sample = next(iter(ds)) @@ -1679,10 +1676,6 @@ def test_wav2vec2_with_lm(self): @require_pyctcdecode @require_torchaudio - @unittest.skipIf( - is_torch_less_than_1_9, - reason="`torchaudio.functional.resample` needs torchaudio >= 0.9 which requires torch >= 0.9", - ) def test_wav2vec2_with_lm_pool(self): ds = load_dataset("common_voice", "es", split="test", streaming=True) sample = next(iter(ds)) diff --git a/tests/onnx/test_onnx_v2.py b/tests/onnx/test_onnx_v2.py index 51d9e2cb1843..81e28a3796a7 100644 --- a/tests/onnx/test_onnx_v2.py +++ b/tests/onnx/test_onnx_v2.py @@ -14,7 +14,6 @@ OnnxConfig, OnnxConfigWithPast, ParameterFormat, - export, validate_model_outputs, ) from transformers.onnx.utils import ( @@ -40,15 +39,6 @@ class OnnxUtilsTestCaseV2(TestCase): Cover all the utilities involved to export ONNX models """ - @require_torch - @patch("transformers.onnx.convert.is_torch_onnx_dict_inputs_support_available", return_value=False) - def test_ensure_pytorch_version_ge_1_8_0(self, mock_is_torch_onnx_dict_inputs_support_available): - """ - Ensure we raise an Exception if the pytorch version is unsupported (< 1.8.0) - """ - self.assertRaises(AssertionError, export, None, None, None, None, None) - mock_is_torch_onnx_dict_inputs_support_available.assert_called() - def test_compute_effective_axis_dimension(self): """ When exporting ONNX model with dynamic axis (batch or sequence) we set batch_size and/or sequence_length = -1. From 5d15c1b4d08370a3f5babeefb7ee7fb41fd84ca0 Mon Sep 17 00:00:00 2001 From: Yanming W Date: Tue, 21 Mar 2023 13:32:43 -0700 Subject: [PATCH 150/277] Restore fp16 support on xla gpu device (#22300) --- src/transformers/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 7c7d5df0b6a2..1b3dc8055e4d 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -598,7 +598,7 @@ def __init__( logger.info(f"Using {args.half_precision_backend} half precision backend") self.do_grad_scaling = False - if (args.fp16 or args.bf16) and not (args.deepspeed or is_sagemaker_mp_enabled() or is_torch_tpu_available()): + if (args.fp16 or args.bf16) and not (args.deepspeed or is_sagemaker_mp_enabled()): # deepspeed and SageMaker Model Parallel manage their own half precision if args.half_precision_backend == "cuda_amp": self.use_cuda_amp = True From a644393fe97f266a35b2173859a2d20b214f7822 Mon Sep 17 00:00:00 2001 From: Ali Hassani <68103095+alihassanijr@users.noreply.github.com> Date: Tue, 21 Mar 2023 14:21:34 -0700 Subject: [PATCH 151/277] Correct NATTEN function signatures and force new version (#22298) --- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- src/transformers/models/dinat/modeling_dinat.py | 2 +- src/transformers/models/nat/modeling_nat.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 70197c209f0f..bddf49e46459 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,7 @@ "keras-nlp>=0.3.1", "librosa", "nltk", - "natten>=0.14.5", + "natten>=0.14.6", "numpy>=1.17", "onnxconverter-common", "onnxruntime-tools>=1.4.2", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index aa23c1ad06cc..c0cacd7596eb 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -35,7 +35,7 @@ "keras-nlp": "keras-nlp>=0.3.1", "librosa": "librosa", "nltk": "nltk", - "natten": "natten>=0.14.5", + "natten": "natten>=0.14.6", "numpy": "numpy>=1.17", "onnxconverter-common": "onnxconverter-common", "onnxruntime-tools": "onnxruntime-tools>=1.4.2", diff --git a/src/transformers/models/dinat/modeling_dinat.py b/src/transformers/models/dinat/modeling_dinat.py index efeb68846fce..2cacab8ac935 100644 --- a/src/transformers/models/dinat/modeling_dinat.py +++ b/src/transformers/models/dinat/modeling_dinat.py @@ -356,7 +356,7 @@ def forward( # seem a bit unusual, but is taken from the original Transformer paper. attention_probs = self.dropout(attention_probs) - context_layer = natten2dav(attention_probs, value_layer, self.dilation) + context_layer = natten2dav(attention_probs, value_layer, self.kernel_size, self.dilation) context_layer = context_layer.permute(0, 2, 3, 1, 4).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) context_layer = context_layer.view(new_context_layer_shape) diff --git a/src/transformers/models/nat/modeling_nat.py b/src/transformers/models/nat/modeling_nat.py index 3a93b81e4bc5..dfe801d19961 100644 --- a/src/transformers/models/nat/modeling_nat.py +++ b/src/transformers/models/nat/modeling_nat.py @@ -348,7 +348,7 @@ def forward( # seem a bit unusual, but is taken from the original Transformer paper. attention_probs = self.dropout(attention_probs) - context_layer = natten2dav(attention_probs, value_layer, 1) + context_layer = natten2dav(attention_probs, value_layer, self.kernel_size, 1) context_layer = context_layer.permute(0, 2, 3, 1, 4).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) context_layer = context_layer.view(new_context_layer_shape) From e3eb2224db8f619bf69ceaa559703ab9f77c2167 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 21 Mar 2023 17:00:05 -0700 Subject: [PATCH 152/277] [deepspeed] offload + non-cpuadam optimizer exception doc (#22044) * [deepspeed] offload + non-cpuadam optimizer exception doc * deps --- docs/source/en/main_classes/deepspeed.mdx | 13 +++++++++++-- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/source/en/main_classes/deepspeed.mdx b/docs/source/en/main_classes/deepspeed.mdx index 3ee96f0b0413..2b5ea6a74354 100644 --- a/docs/source/en/main_classes/deepspeed.mdx +++ b/docs/source/en/main_classes/deepspeed.mdx @@ -1293,8 +1293,17 @@ If you want to use another optimizer which is not listed above, you will have to } ``` -Similarly to `AdamW`, you can configure other officially supported optimizers. Just remember that may have different -config values. e.g. for Adam you will want `weight_decay` around `0.01`. +Similarly to `AdamW`, you can configure other officially supported optimizers. Just remember that those may have different config values. e.g. for Adam you will want `weight_decay` around `0.01`. + +Additionally, offload works the best when it's used with Deepspeed's CPU Adam optimizer. If you want to use a different optimizer with offload, since `deepspeed==0.8.3` you need to also add: + + +```json +{ + "zero_force_ds_cpu_optimizer": false +} +``` +to the top level configuration. diff --git a/setup.py b/setup.py index bddf49e46459..669c8d959caf 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ "dataclasses", "datasets!=2.5.0", "decord==0.6.0", - "deepspeed>=0.6.5", + "deepspeed>=0.8.3", "dill<0.3.5", "evaluate>=0.2.0", "fairscale>0.3", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index c0cacd7596eb..7c596c266830 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -12,7 +12,7 @@ "dataclasses": "dataclasses", "datasets": "datasets!=2.5.0", "decord": "decord==0.6.0", - "deepspeed": "deepspeed>=0.6.5", + "deepspeed": "deepspeed>=0.8.3", "dill": "dill<0.3.5", "evaluate": "evaluate>=0.2.0", "fairscale": "fairscale>0.3", From 9a7924417570e886905659fa1e87adaaa9678d53 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 22 Mar 2023 01:00:33 +0100 Subject: [PATCH 153/277] Final update of doctest (#22299) * update * update --------- Co-authored-by: ydshieh --- .../models/auto/feature_extraction_auto.py | 2 +- .../models/auto/image_processing_auto.py | 2 +- src/transformers/models/auto/processing_auto.py | 2 +- src/transformers/models/auto/tokenization_auto.py | 2 +- .../models/bertweet/tokenization_bertweet.py | 14 +++++++++++--- src/transformers/models/dpr/tokenization_dpr.py | 1 + .../models/dpr/tokenization_dpr_fast.py | 1 + .../tokenization_gptsan_japanese.py | 5 ++++- .../models/m2m_100/tokenization_m2m_100.py | 5 +++-- .../models/marian/tokenization_marian.py | 4 ++-- .../models/roformer/tokenization_roformer.py | 2 +- .../models/roformer/tokenization_roformer_fast.py | 2 +- .../models/transfo_xl/tokenization_transfo_xl.py | 4 ++-- utils/documentation_tests.txt | 13 +++++++++++++ 14 files changed, 43 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index adeadf17e3a7..90218d137f8d 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -303,7 +303,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): >>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h") >>> # If feature extractor files are in a directory (e.g. feature extractor was saved using *save_pretrained('./test/saved_model/')*) - >>> feature_extractor = AutoFeatureExtractor.from_pretrained("./test/saved_model/") + >>> # feature_extractor = AutoFeatureExtractor.from_pretrained("./test/saved_model/") ```""" config = kwargs.pop("config", None) trust_remote_code = kwargs.pop("trust_remote_code", False) diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 7a042c56c5c7..95ecd57172b4 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -306,7 +306,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k") >>> # If image processor files are in a directory (e.g. image processor was saved using *save_pretrained('./test/saved_model/')*) - >>> image_processor = AutoImageProcessor.from_pretrained("./test/saved_model/") + >>> # image_processor = AutoImageProcessor.from_pretrained("./test/saved_model/") ```""" config = kwargs.pop("config", None) trust_remote_code = kwargs.pop("trust_remote_code", False) diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index 29726fde4f08..fb020034008b 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -188,7 +188,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h") >>> # If processor files are in a directory (e.g. processor was saved using *save_pretrained('./test/saved_model/')*) - >>> processor = AutoProcessor.from_pretrained("./test/saved_model/") + >>> # processor = AutoProcessor.from_pretrained("./test/saved_model/") ```""" config = kwargs.pop("config", None) trust_remote_code = kwargs.pop("trust_remote_code", False) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 2e6ddc0b132c..a029bfb52cd6 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -575,7 +575,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*) - >>> tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/") + >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/") >>> # Download vocabulary from huggingface.co and define model-specific arguments >>> tokenizer = AutoTokenizer.from_pretrained("roberta-base", add_prefix_space=True) diff --git a/src/transformers/models/bertweet/tokenization_bertweet.py b/src/transformers/models/bertweet/tokenization_bertweet.py index 129806ebd3b8..9a5390c0b706 100644 --- a/src/transformers/models/bertweet/tokenization_bertweet.py +++ b/src/transformers/models/bertweet/tokenization_bertweet.py @@ -640,9 +640,17 @@ def _replace_html_entities(text, keep=(), remove_illegal=True, encoding="utf-8") See https://github.com/scrapy/w3lib/blob/master/w3lib/html.py - >>> from nltk.tokenize.casual import _replace_html_entities >>> _replace_html_entities(b'Price: £100') - 'Price: \\xa3100' >>> print(_replace_html_entities(b'Price: £100')) Price: £100 >>> - """ + Examples: + + ```python + >>> from nltk.tokenize.casual import _replace_html_entities + + >>> _replace_html_entities(b"Price: £100") + 'Price: \\xa3100' + + >>> print(_replace_html_entities(b"Price: £100")) + Price: £100 + ```""" def _convert_entity(match): entity_body = match.group(3) diff --git a/src/transformers/models/dpr/tokenization_dpr.py b/src/transformers/models/dpr/tokenization_dpr.py index a14133459b7e..a2024dda5d69 100644 --- a/src/transformers/models/dpr/tokenization_dpr.py +++ b/src/transformers/models/dpr/tokenization_dpr.py @@ -316,6 +316,7 @@ def decode_best_spans( >>> outputs = model(**encoded_inputs) >>> predicted_spans = tokenizer.decode_best_spans(encoded_inputs, outputs) >>> print(predicted_spans[0].text) # best span + a song ```""" input_ids = reader_input["input_ids"] start_logits, end_logits, relevance_logits = reader_output[:3] diff --git a/src/transformers/models/dpr/tokenization_dpr_fast.py b/src/transformers/models/dpr/tokenization_dpr_fast.py index 507cd2bc40bc..de32332bf271 100644 --- a/src/transformers/models/dpr/tokenization_dpr_fast.py +++ b/src/transformers/models/dpr/tokenization_dpr_fast.py @@ -316,6 +316,7 @@ def decode_best_spans( >>> outputs = model(**encoded_inputs) >>> predicted_spans = tokenizer.decode_best_spans(encoded_inputs, outputs) >>> print(predicted_spans[0].text) # best span + a song ```""" input_ids = reader_input["input_ids"] start_logits, end_logits, relevance_logits = reader_output[:3] diff --git a/src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py b/src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py index a9ebc21f3843..0c89a60b786f 100644 --- a/src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py +++ b/src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py @@ -96,7 +96,7 @@ class GPTSanJapaneseTokenizer(PreTrainedTokenizer): >>> tokenizer = GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese") >>> # You can confirm both 慶応 and 慶應 are encoded to 17750 >>> tokenizer("吾輩は猫である🐯。実は慶応(慶應)大学出身")["input_ids"] - [34347, 31459, 30647, 31448, 25, 30659, 35729, 35676, 32417, 30647, 17750, 35589, 17750, 35590, 321, 1281] + [35993, 35998, 34347, 31459, 30647, 31448, 25, 30659, 35729, 35676, 32417, 30647, 17750, 35589, 17750, 35590, 321, 1281] >>> # Both 慶応 and 慶應 are decoded to 慶応 >>> tokenizer.decode(tokenizer("吾輩は猫である🐯。実は慶応(慶應)大学出身")["input_ids"]) @@ -311,6 +311,9 @@ def create_token_type_ids_from_sequences( Example: ```python + >>> from transformers import GPTSanJapaneseTokenizer + + >>> tokenizer = GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese") >>> x_token = tokenizer("ｱｲｳｴ") >>> # input_ids: | SOT | SEG | ｱ | ｲ | ｳ | ｴ | >>> # token_type_ids: | 1 | 0 | 0 | 0 | 0 | 0 | diff --git a/src/transformers/models/m2m_100/tokenization_m2m_100.py b/src/transformers/models/m2m_100/tokenization_m2m_100.py index dcfa51555fc3..82f5e3a47b36 100644 --- a/src/transformers/models/m2m_100/tokenization_m2m_100.py +++ b/src/transformers/models/m2m_100/tokenization_m2m_100.py @@ -110,13 +110,14 @@ class M2M100Tokenizer(PreTrainedTokenizer): Examples: ```python - >>> from transformers import M2M100Tokenizer + >>> from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer + >>> model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") >>> tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M", src_lang="en", tgt_lang="ro") >>> src_text = " UN Chief Says There Is No Military Solution in Syria" >>> tgt_text = "Şeful ONU declară că nu există o soluţie militară în Siria" >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors="pt") - >>> model(**model_inputs) # should work + >>> outputs = model(**model_inputs) # should work ```""" vocab_files_names = VOCAB_FILES_NAMES diff --git a/src/transformers/models/marian/tokenization_marian.py b/src/transformers/models/marian/tokenization_marian.py index 7d2af76fc331..aa63b1ff88aa 100644 --- a/src/transformers/models/marian/tokenization_marian.py +++ b/src/transformers/models/marian/tokenization_marian.py @@ -106,13 +106,13 @@ class MarianTokenizer(PreTrainedTokenizer): Examples: ```python - >>> from transformers import MarianTokenizer + >>> from transformers import MarianForCausalLM, MarianTokenizer + >>> model = MarianForCausalLM.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> src_texts = ["I am a small frog.", "Tom asked his teacher for advice."] >>> tgt_texts = ["Ich bin ein kleiner Frosch.", "Tom bat seinen Lehrer um Rat."] # optional >>> inputs = tokenizer(src_texts, text_target=tgt_texts, return_tensors="pt", padding=True) - # keys [input_ids, attention_mask, labels]. >>> outputs = model(**inputs) # should work ```""" diff --git a/src/transformers/models/roformer/tokenization_roformer.py b/src/transformers/models/roformer/tokenization_roformer.py index 6c0b6cd4f3c2..1e4907f50c58 100644 --- a/src/transformers/models/roformer/tokenization_roformer.py +++ b/src/transformers/models/roformer/tokenization_roformer.py @@ -344,7 +344,7 @@ class RoFormerTokenizer(PreTrainedTokenizer): >>> tokenizer = RoFormerTokenizer.from_pretrained("junnyu/roformer_chinese_base") >>> tokenizer.tokenize("今天天气非常好。") - # ['今', '天', '天', '气', '非常', '好', '。'] + ['今', '天', '天', '气', '非常', '好', '。'] ```""" vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP diff --git a/src/transformers/models/roformer/tokenization_roformer_fast.py b/src/transformers/models/roformer/tokenization_roformer_fast.py index 88ccf183d174..d73e3cdb93cc 100644 --- a/src/transformers/models/roformer/tokenization_roformer_fast.py +++ b/src/transformers/models/roformer/tokenization_roformer_fast.py @@ -85,7 +85,7 @@ class RoFormerTokenizerFast(PreTrainedTokenizerFast): >>> tokenizer = RoFormerTokenizerFast.from_pretrained("junnyu/roformer_chinese_base") >>> tokenizer.tokenize("今天天气非常好。") - # ['今', '天', '天', '气', '非常', '好', '。'] + ['今', '天', '天', '气', '非常', '好', '。'] ```""" vocab_files_names = VOCAB_FILES_NAMES diff --git a/src/transformers/models/transfo_xl/tokenization_transfo_xl.py b/src/transformers/models/transfo_xl/tokenization_transfo_xl.py index 13977d438280..0097b2a6f20d 100644 --- a/src/transformers/models/transfo_xl/tokenization_transfo_xl.py +++ b/src/transformers/models/transfo_xl/tokenization_transfo_xl.py @@ -88,7 +88,7 @@ def tokenize_numbers(text_array: List[str]) -> List[str]: ```python >>> tokenize_numbers(["$", "5,000", "1.73", "m"]) - ["$", "5", "@,@", "000", "1", "@.@", "73", "m"] + ['$', '5', '@,@', '000', '1', '@.@', '73', 'm'] ```""" tokenized = [] for i in range(len(text_array)): @@ -113,7 +113,7 @@ def detokenize_numbers(text: str) -> str: ```python >>> detokenize_numbers("$ 5 @,@ 000 1 @.@ 73 m") - "$ 5,000 1.73 m" + '$ 5,000 1.73 m' ```""" for reg, sub in DETOKENIZE_NUMBERS: text = re.sub(reg, sub, text) diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index bfd87a5c0be1..3357c1d56952 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -467,3 +467,16 @@ src/transformers/models/mvp/tokenization_mvp.py src/transformers/models/mvp/tokenization_mvp_fast.py src/transformers/models/roberta/tokenization_roberta.py src/transformers/models/roberta/tokenization_roberta_fast.py +src/transformers/models/auto/feature_extraction_auto.py +src/transformers/models/auto/image_processing_auto.py +src/transformers/models/auto/processing_auto.py +src/transformers/models/auto/tokenization_auto.py +src/transformers/models/bertweet/tokenization_bertweet.py +src/transformers/models/dpr/tokenization_dpr.py +src/transformers/models/dpr/tokenization_dpr_fast.py +src/transformers/models/gptsan_japanese/tokenization_gptsan_japanese.py +src/transformers/models/m2m_100/tokenization_m2m_100.py +src/transformers/models/marian/tokenization_marian.py +src/transformers/models/roformer/tokenization_roformer.py +src/transformers/models/roformer/tokenization_roformer_fast.py +src/transformers/models/transfo_xl/tokenization_transfo_xl.py \ No newline at end of file From 79468e5f442bf01521f9199742c1be7c12b32eb1 Mon Sep 17 00:00:00 2001 From: Alara Dirik <8944735+alaradirik@users.noreply.github.com> Date: Wed, 22 Mar 2023 07:35:47 +0300 Subject: [PATCH 154/277] Add MaskedImageModelingOutput (#22212) * Add MaskedImageModelingOutput --- src/transformers/modeling_outputs.py | 38 +++++++++++++++++ src/transformers/modeling_tf_outputs.py | 42 ++++++++++++++++++- src/transformers/models/deit/modeling_deit.py | 17 +++++--- .../models/deit/modeling_tf_deit.py | 18 ++++---- src/transformers/models/vit/modeling_vit.py | 17 +++++--- tests/models/deit/test_modeling_deit.py | 4 +- tests/models/deit/test_modeling_tf_deit.py | 4 +- tests/models/vit/test_modeling_vit.py | 4 +- 8 files changed, 116 insertions(+), 28 deletions(-) diff --git a/src/transformers/modeling_outputs.py b/src/transformers/modeling_outputs.py index 4f7540d0ff9e..c69e426ab531 100755 --- a/src/transformers/modeling_outputs.py +++ b/src/transformers/modeling_outputs.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings from dataclasses import dataclass from typing import Optional, Tuple @@ -1622,3 +1623,40 @@ class SampleTSPredictionOutput(ModelOutput): """ sequences: torch.FloatTensor = None + + +@dataclass +class MaskedImageModelingOutput(ModelOutput): + """ + Base class for outputs of masked image completion / in-painting models. + + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided): + Reconstruction loss. + reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Reconstructed / completed images. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or + when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each stage) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states + (also called feature maps) of the model at the output of each stage. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when + `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, + sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in + the self-attention heads. + """ + + loss: Optional[torch.FloatTensor] = None + reconstruction: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + @property + def logits(self): + warnings.warn( + "logits attribute is deprecated and will be removed in version 5 of Transformers." + " Please use the reconstruction attribute to retrieve the final output instead.", + FutureWarning, + ) + return self.reconstruction diff --git a/src/transformers/modeling_tf_outputs.py b/src/transformers/modeling_tf_outputs.py index 0fed3e78511a..f8148b169543 100644 --- a/src/transformers/modeling_tf_outputs.py +++ b/src/transformers/modeling_tf_outputs.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings from dataclasses import dataclass from typing import List, Optional, Tuple @@ -55,8 +56,8 @@ class TFBaseModelOutputWithNoAttention(ModelOutput): last_hidden_state (`tf.Tensor` shape `(batch_size, num_channels, height, width)`): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + - one for the output of each layer) of shape `(batch_size, num_channels, height, width)`. + Tuple of `tf.Tensor` (one for the output of the embeddings, if the model has an embedding layer, + one for + the output of each layer) of shape `(batch_size, num_channels, height, width)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. """ @@ -949,3 +950,40 @@ class TFImageClassifierOutputWithNoAttention(ModelOutput): loss: Optional[tf.Tensor] = None logits: tf.Tensor = None hidden_states: Optional[Tuple[tf.Tensor, ...]] = None + + +@dataclass +class TFMaskedImageModelingOutput(ModelOutput): + """ + Base class for outputs of masked image completion / in-painting models. + + Args: + loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided): + Reconstruction loss. + reconstruction (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`): + Reconstructed / completed images. + hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when + `config.output_hidden_states=True`): + Tuple of `tf.Tensor` (one for the output of the embeddings, if the model has an embedding layer, + one for + the output of each stage) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states (also called + feature maps) of the model at the output of each stage. + attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when + `config.output_attentions=True`): + Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, patch_size, sequence_length)`. + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + loss: Optional[tf.Tensor] = None + reconstruction: tf.Tensor = None + hidden_states: Optional[Tuple[tf.Tensor]] = None + attentions: Optional[Tuple[tf.Tensor]] = None + + @property + def logits(self): + warnings.warn( + "logits attribute is deprecated and will be removed in version 5 of Transformers." + " Please use the reconstruction attribute to retrieve the final output instead.", + FutureWarning, + ) + return self.reconstruction diff --git a/src/transformers/models/deit/modeling_deit.py b/src/transformers/models/deit/modeling_deit.py index 2a37e3ebc1f6..b5d1aa28c918 100644 --- a/src/transformers/models/deit/modeling_deit.py +++ b/src/transformers/models/deit/modeling_deit.py @@ -26,7 +26,12 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput, MaskedLMOutput +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPooling, + ImageClassifierOutput, + MaskedImageModelingOutput, +) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( @@ -592,7 +597,7 @@ def __init__(self, config: DeiTConfig) -> None: self.post_init() @add_start_docstrings_to_model_forward(DEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @replace_return_docstrings(output_type=MaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -601,7 +606,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[tuple, MaskedLMOutput]: + ) -> Union[tuple, MaskedImageModelingOutput]: r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). @@ -627,7 +632,7 @@ def forward( >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool() >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos) - >>> loss, reconstructed_pixel_values = outputs.loss, outputs.logits + >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction >>> list(reconstructed_pixel_values.shape) [1, 3, 224, 224] ```""" @@ -670,9 +675,9 @@ def forward( output = (reconstructed_pixel_values,) + outputs[1:] return ((masked_im_loss,) + output) if masked_im_loss is not None else output - return MaskedLMOutput( + return MaskedImageModelingOutput( loss=masked_im_loss, - logits=reconstructed_pixel_values, + reconstruction=reconstructed_pixel_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) diff --git a/src/transformers/models/deit/modeling_tf_deit.py b/src/transformers/models/deit/modeling_tf_deit.py index 1bcfd07b38ff..558de60614e0 100644 --- a/src/transformers/models/deit/modeling_tf_deit.py +++ b/src/transformers/models/deit/modeling_tf_deit.py @@ -27,7 +27,7 @@ TFBaseModelOutput, TFBaseModelOutputWithPooling, TFImageClassifierOutput, - TFMaskedLMOutput, + TFMaskedImageModelingOutput, ) from ...modeling_tf_utils import ( TFPreTrainedModel, @@ -769,7 +769,7 @@ def __init__(self, config: DeiTConfig) -> None: @unpack_inputs @add_start_docstrings_to_model_forward(DEIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TFMaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @replace_return_docstrings(output_type=TFMaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) def call( self, pixel_values: Optional[tf.Tensor] = None, @@ -779,7 +779,7 @@ def call( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, training: bool = False, - ) -> Union[tuple, TFMaskedLMOutput]: + ) -> Union[tuple, TFMaskedImageModelingOutput]: r""" bool_masked_pos (`tf.Tensor` of type bool and shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). @@ -805,7 +805,7 @@ def call( >>> bool_masked_pos = tf.cast(tf.random.uniform((1, num_patches), minval=0, maxval=2, dtype=tf.int32), tf.bool) >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos) - >>> loss, reconstructed_pixel_values = outputs.loss, outputs.logits + >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction >>> list(reconstructed_pixel_values.shape) [1, 3, 224, 224] ```""" @@ -860,18 +860,20 @@ def call( output = (reconstructed_pixel_values,) + outputs[1:] return ((masked_im_loss,) + output) if masked_im_loss is not None else output - return TFMaskedLMOutput( + return TFMaskedImageModelingOutput( loss=masked_im_loss, - logits=reconstructed_pixel_values, + reconstruction=reconstructed_pixel_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) - def serving_output(self, output: TFMaskedLMOutput) -> TFMaskedLMOutput: + def serving_output(self, output: TFMaskedImageModelingOutput) -> TFMaskedImageModelingOutput: hidden_states = tf.convert_to_tensor(output.hidden_states) if self.config.output_hidden_states else None attentions = tf.convert_to_tensor(output.attentions) if self.config.output_attentions else None - return TFMaskedLMOutput(logits=output.logits, hidden_states=hidden_states, attentions=attentions) + return TFMaskedImageModelingOutput( + reconstruction=output.reconstruction, hidden_states=hidden_states, attentions=attentions + ) @add_start_docstrings( diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index 03a9212ff03f..eba8278eaafe 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -25,7 +25,12 @@ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, ImageClassifierOutput, MaskedLMOutput +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPooling, + ImageClassifierOutput, + MaskedImageModelingOutput, +) from ...modeling_utils import PreTrainedModel from ...pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer from ...utils import ( @@ -647,7 +652,7 @@ def __init__(self, config: ViTConfig) -> None: self.post_init() @add_start_docstrings_to_model_forward(VIT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + @replace_return_docstrings(output_type=MaskedImageModelingOutput, config_class=_CONFIG_FOR_DOC) def forward( self, pixel_values: Optional[torch.Tensor] = None, @@ -657,7 +662,7 @@ def forward( output_hidden_states: Optional[bool] = None, interpolate_pos_encoding: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[tuple, MaskedLMOutput]: + ) -> Union[tuple, MaskedImageModelingOutput]: r""" bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`): Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). @@ -683,7 +688,7 @@ def forward( >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool() >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos) - >>> loss, reconstructed_pixel_values = outputs.loss, outputs.logits + >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction >>> list(reconstructed_pixel_values.shape) [1, 3, 224, 224] ```""" @@ -727,9 +732,9 @@ def forward( output = (reconstructed_pixel_values,) + outputs[1:] return ((masked_im_loss,) + output) if masked_im_loss is not None else output - return MaskedLMOutput( + return MaskedImageModelingOutput( loss=masked_im_loss, - logits=reconstructed_pixel_values, + reconstruction=reconstructed_pixel_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) diff --git a/tests/models/deit/test_modeling_deit.py b/tests/models/deit/test_modeling_deit.py index fbf6d7353b07..1564b23aa659 100644 --- a/tests/models/deit/test_modeling_deit.py +++ b/tests/models/deit/test_modeling_deit.py @@ -145,7 +145,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label model.eval() result = model(pixel_values) self.parent.assertEqual( - result.logits.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) + result.reconstruction.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) ) # test greyscale images @@ -156,7 +156,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size]) result = model(pixel_values) - self.parent.assertEqual(result.logits.shape, (self.batch_size, 1, self.image_size, self.image_size)) + self.parent.assertEqual(result.reconstruction.shape, (self.batch_size, 1, self.image_size, self.image_size)) def create_and_check_for_image_classification(self, config, pixel_values, labels): config.num_labels = self.type_sequence_label_size diff --git a/tests/models/deit/test_modeling_tf_deit.py b/tests/models/deit/test_modeling_tf_deit.py index c7c1fc84568b..223d164d4aaf 100644 --- a/tests/models/deit/test_modeling_tf_deit.py +++ b/tests/models/deit/test_modeling_tf_deit.py @@ -130,7 +130,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label model = TFDeiTForMaskedImageModeling(config=config) result = model(pixel_values) self.parent.assertEqual( - result.logits.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) + result.reconstruction.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) ) # test greyscale images @@ -139,7 +139,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size]) result = model(pixel_values) - self.parent.assertEqual(result.logits.shape, (self.batch_size, 1, self.image_size, self.image_size)) + self.parent.assertEqual(result.reconstruction.shape, (self.batch_size, 1, self.image_size, self.image_size)) def create_and_check_for_image_classification(self, config, pixel_values, labels): config.num_labels = self.type_sequence_label_size diff --git a/tests/models/vit/test_modeling_vit.py b/tests/models/vit/test_modeling_vit.py index be509d460e63..d7ae3c162f59 100644 --- a/tests/models/vit/test_modeling_vit.py +++ b/tests/models/vit/test_modeling_vit.py @@ -134,7 +134,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label model.eval() result = model(pixel_values) self.parent.assertEqual( - result.logits.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) + result.reconstruction.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size) ) # test greyscale images @@ -145,7 +145,7 @@ def create_and_check_for_masked_image_modeling(self, config, pixel_values, label pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size]) result = model(pixel_values) - self.parent.assertEqual(result.logits.shape, (self.batch_size, 1, self.image_size, self.image_size)) + self.parent.assertEqual(result.reconstruction.shape, (self.batch_size, 1, self.image_size, self.image_size)) def create_and_check_for_image_classification(self, config, pixel_values, labels): config.num_labels = self.type_sequence_label_size From 378ce252a7db03cc9ed02f068d3426f3ee04c08d Mon Sep 17 00:00:00 2001 From: jiqing-feng <107918818+jiqing-feng@users.noreply.github.com> Date: Wed, 22 Mar 2023 18:19:26 +0800 Subject: [PATCH 155/277] Enable traced model for text-generation task (#22265) --- .../pytorch/text-generation/run_generation.py | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/examples/pytorch/text-generation/run_generation.py b/examples/pytorch/text-generation/run_generation.py index 9b4b09fc9687..e0dda0ec0c2f 100755 --- a/examples/pytorch/text-generation/run_generation.py +++ b/examples/pytorch/text-generation/run_generation.py @@ -20,6 +20,7 @@ import argparse import logging +from typing import Tuple import numpy as np import torch @@ -27,6 +28,7 @@ from transformers import ( CTRLLMHeadModel, CTRLTokenizer, + GenerationMixin, GPT2LMHeadModel, GPT2Tokenizer, OpenAIGPTLMHeadModel, @@ -38,6 +40,7 @@ XLNetLMHeadModel, XLNetTokenizer, ) +from transformers.modeling_outputs import CausalLMOutputWithPast logging.basicConfig( @@ -151,6 +154,131 @@ def adjust_length_to_model(length, max_sequence_length): return length +def sparse_model_config(model_config): + embedding_size = None + if hasattr(model_config, "hidden_size"): + embedding_size = model_config.hidden_size + elif hasattr(model_config, "n_embed"): + embedding_size = model_config.n_embed + elif hasattr(model_config, "n_embd"): + embedding_size = model_config.n_embd + + num_head = None + if hasattr(model_config, "num_attention_heads"): + num_head = model_config.num_attention_heads + elif hasattr(model_config, "n_head"): + num_head = model_config.n_head + + if embedding_size is None or num_head is None or num_head == 0: + raise ValueError("Check the model config") + + num_embedding_size_per_head = int(embedding_size / num_head) + num_layer = model_config.n_layer + + return num_layer, num_head, num_embedding_size_per_head + + +def prepare_jit_inputs(inputs, model, tokenizer): + num_batch = len(inputs) + dummy_input = tokenizer.batch_encode_plus(inputs, return_tensors="pt", padding=True) + num_block_layers, num_attention_heads, num_embedding_size_per_head = sparse_model_config(model.config) + if model.config.model_type == "bloom": + past_key_values = tuple( + ( + torch.zeros(int(num_attention_heads * num_batch), num_embedding_size_per_head, 1) + .to(model.config.torch_dtype) + .to(model.device), + torch.zeros(int(num_attention_heads * num_batch), 1, num_embedding_size_per_head) + .to(model.config.torch_dtype) + .to(model.device), + ) + for _ in range(num_block_layers) + ) + else: + past_key_values = tuple( + ( + torch.zeros(num_batch, num_attention_heads, 1, num_embedding_size_per_head) + .to(model.config.torch_dtype) + .to(model.device), + torch.zeros(num_batch, num_attention_heads, 1, num_embedding_size_per_head) + .to(model.config.torch_dtype) + .to(model.device), + ) + for _ in range(num_block_layers) + ) + + dummy_input["attention_mask"] = torch.cat( + [ + torch.zeros(dummy_input["attention_mask"].shape[0], 1).to(dummy_input["attention_mask"].dtype), + dummy_input["attention_mask"], + ], + -1, + ) + + if model.config.use_cache: + jit_inputs = ( + dummy_input["input_ids"].to(model.device), + past_key_values, + dummy_input["attention_mask"].to(model.device), + ) + else: + jit_inputs = ( + dummy_input["input_ids"].to(model.device), + dummy_input["attention_mask"].to(model.device), + ) + + return jit_inputs + + +class _ModelFallbackWrapper(GenerationMixin): + __slots__ = ("_optimized", "_default") + + def __init__(self, optimized, default): + self._optimized = optimized + self._default = default + + def __call__(self, *args, **kwargs): + if kwargs["past_key_values"] is None: + return self._default(*args, **kwargs) + trace_graph_inputs = [] + kwargs.pop("position_ids", None) + for k, v in kwargs.items(): + if v is not None and not isinstance(v, bool): + trace_graph_inputs.append(v) + trace_graph_inputs = tuple(trace_graph_inputs) + outputs = self._optimized(*trace_graph_inputs) + lm_logits = outputs[0] + past_key_values = outputs[1] + fixed_output = CausalLMOutputWithPast( + loss=None, + logits=lm_logits, + past_key_values=past_key_values, + hidden_states=None, + attentions=None, + ) + return fixed_output + + def __getattr__(self, item): + return getattr(self._default, item) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, inputs_embeds=None, use_cache=None, **kwargs + ): + return self._default.prepare_inputs_for_generation( + input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, **kwargs + ) + + def _reorder_cache( + self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor + ) -> Tuple[Tuple[torch.Tensor]]: + """ + This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or + [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct + beam_idx at every generation step. + """ + return self._default._reorder_cache(past_key_values, beam_idx) + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -196,6 +324,9 @@ def main(): action="store_true", help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit", ) + parser.add_argument( + "--jit", type=bool, default=False, help="Whether or not to use jit trace to accelerate inference" + ) args = parser.parse_args() args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") @@ -213,6 +344,8 @@ def main(): raise KeyError("the model {} you specified is not supported. You are welcome to add it and open a PR :)") tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token model = model_class.from_pretrained(args.model_name_or_path) model.to(args.device) @@ -248,6 +381,18 @@ def main(): else: input_ids = encoded_prompt + if args.jit: + jit_input_texts = ["jit"] + jit_inputs = prepare_jit_inputs(jit_input_texts, model, tokenizer) + torch._C._jit_set_texpr_fuser_enabled(False) + model.config.return_dict = False + traced_model = torch.jit.trace(model, jit_inputs, strict=False) + traced_model = torch.jit.freeze(traced_model.eval()) + traced_model(*jit_inputs) + traced_model(*jit_inputs) + + model = _ModelFallbackWrapper(traced_model, model) + output_sequences = model.generate( input_ids=input_ids, max_length=args.length + len(encoded_prompt[0]), From 2c37c0fdd211e7b50458254d8b9119dcda8bdfed Mon Sep 17 00:00:00 2001 From: "Wang, Yi" Date: Wed, 22 Mar 2023 18:42:39 +0800 Subject: [PATCH 156/277] =?UTF-8?q?add=20low=5Fcpu=5Fmem=5Fusage=20option?= =?UTF-8?q?=20in=20run=5Fclm.py=20example=20which=20will=20benefit?= =?UTF-8?q?=E2=80=A6=20(#22288)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add low_cpu_mem_usage option in run_clm.py example which will benefit LLM loading Signed-off-by: Wang, Yi A * update all the example and README under language-modeling Signed-off-by: Wang, Yi A --------- Signed-off-by: Wang, Yi A --- examples/pytorch/language-modeling/README.md | 4 ++++ examples/pytorch/language-modeling/run_clm.py | 10 ++++++++++ .../pytorch/language-modeling/run_clm_no_trainer.py | 9 +++++++++ examples/pytorch/language-modeling/run_mlm.py | 10 ++++++++++ .../pytorch/language-modeling/run_mlm_no_trainer.py | 9 +++++++++ examples/pytorch/language-modeling/run_plm.py | 10 ++++++++++ 6 files changed, 52 insertions(+) diff --git a/examples/pytorch/language-modeling/README.md b/examples/pytorch/language-modeling/README.md index ff504b535747..3069fe9eb974 100644 --- a/examples/pytorch/language-modeling/README.md +++ b/examples/pytorch/language-modeling/README.md @@ -178,6 +178,10 @@ sure all your batches have the same length. To use the streaming dataset mode which can be very useful for large datasets, add `--streaming` to the command line. This is currently supported by `run_mlm.py` and `run_clm.py`. +## Low Cpu Memory Usage + +To use low cpu memory mode which can be very useful for LLM, add `--low_cpu_mem_usage` to the command line. This is currently supported by `run_clm.py`,`run_mlm.py`, `run_plm.py`,`run_mlm_no_trainer.py` and `run_clm_no_trainer.py`. + ## Creating a model on the fly When training a model from scratch, configuration values may be overridden with the help of `--config_overrides`: diff --git a/examples/pytorch/language-modeling/run_clm.py b/examples/pytorch/language-modeling/run_clm.py index 9b83c8e2ee57..619697f9b49f 100755 --- a/examples/pytorch/language-modeling/run_clm.py +++ b/examples/pytorch/language-modeling/run_clm.py @@ -130,6 +130,15 @@ class ModelArguments: "choices": ["auto", "bfloat16", "float16", "float32"], }, ) + low_cpu_mem_usage: bool = field( + default=False, + metadata={ + "help": ( + "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded." + "set True will benefit LLM loading time and RAM consumption." + ) + }, + ) def __post_init__(self): if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None): @@ -408,6 +417,7 @@ def main(): revision=model_args.model_revision, use_auth_token=True if model_args.use_auth_token else None, torch_dtype=torch_dtype, + low_cpu_mem_usage=model_args.low_cpu_mem_usage, ) else: model = AutoModelForCausalLM.from_config(config) diff --git a/examples/pytorch/language-modeling/run_clm_no_trainer.py b/examples/pytorch/language-modeling/run_clm_no_trainer.py index e43ff3855536..a62a0267bf19 100755 --- a/examples/pytorch/language-modeling/run_clm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_clm_no_trainer.py @@ -220,6 +220,14 @@ def parse_args(): "Only applicable when `--with_tracking` is passed." ), ) + parser.add_argument( + "--low_cpu_mem_usage", + action="store_true", + help=( + "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded." + "If passed, LLM loading time and RAM consumption will be benefited." + ), + ) args = parser.parse_args() # Sanity checks @@ -374,6 +382,7 @@ def main(): args.model_name_or_path, from_tf=bool(".ckpt" in args.model_name_or_path), config=config, + low_cpu_mem_usage=args.low_cpu_mem_usage, ) else: logger.info("Training new model from scratch") diff --git a/examples/pytorch/language-modeling/run_mlm.py b/examples/pytorch/language-modeling/run_mlm.py index a8d3d8f29aef..b8336fedce08 100755 --- a/examples/pytorch/language-modeling/run_mlm.py +++ b/examples/pytorch/language-modeling/run_mlm.py @@ -116,6 +116,15 @@ class ModelArguments: ) }, ) + low_cpu_mem_usage: bool = field( + default=False, + metadata={ + "help": ( + "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded." + "set True will benefit LLM loading time and RAM consumption." + ) + }, + ) def __post_init__(self): if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None): @@ -395,6 +404,7 @@ def main(): cache_dir=model_args.cache_dir, revision=model_args.model_revision, use_auth_token=True if model_args.use_auth_token else None, + low_cpu_mem_usage=model_args.low_cpu_mem_usage, ) else: logger.info("Training new model from scratch") diff --git a/examples/pytorch/language-modeling/run_mlm_no_trainer.py b/examples/pytorch/language-modeling/run_mlm_no_trainer.py index 78cf3134997d..c67535edb267 100755 --- a/examples/pytorch/language-modeling/run_mlm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_mlm_no_trainer.py @@ -227,6 +227,14 @@ def parse_args(): "Only applicable when `--with_tracking` is passed." ), ) + parser.add_argument( + "--low_cpu_mem_usage", + action="store_true", + help=( + "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded." + "If passed, LLM loading time and RAM consumption will be benefited." + ), + ) args = parser.parse_args() # Sanity checks @@ -379,6 +387,7 @@ def main(): args.model_name_or_path, from_tf=bool(".ckpt" in args.model_name_or_path), config=config, + low_cpu_mem_usage=args.low_cpu_mem_usage, ) else: logger.info("Training new model from scratch") diff --git a/examples/pytorch/language-modeling/run_plm.py b/examples/pytorch/language-modeling/run_plm.py index 582fe5215bb5..06ab1f29f86f 100755 --- a/examples/pytorch/language-modeling/run_plm.py +++ b/examples/pytorch/language-modeling/run_plm.py @@ -104,6 +104,15 @@ class ModelArguments: ) }, ) + low_cpu_mem_usage: bool = field( + default=False, + metadata={ + "help": ( + "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded." + "set True will benefit LLM loading time and RAM consumption." + ) + }, + ) def __post_init__(self): if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None): @@ -375,6 +384,7 @@ def main(): cache_dir=model_args.cache_dir, revision=model_args.model_revision, use_auth_token=True if model_args.use_auth_token else None, + low_cpu_mem_usage=model_args.low_cpu_mem_usage, ) else: logger.info("Training new model from scratch") From 9b1c935cd623421e20890b9b17500695aa8f1f21 Mon Sep 17 00:00:00 2001 From: Connor Henderson Date: Wed, 22 Mar 2023 06:46:56 -0400 Subject: [PATCH 157/277] fix: Allow only test_file in pytorch and flax summarization (#22293) allow only test_file in pytorch and flax summarization --- .../summarization/run_summarization_flax.py | 24 ++++++++++++------- .../summarization/run_summarization.py | 24 ++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/examples/flax/summarization/run_summarization_flax.py b/examples/flax/summarization/run_summarization_flax.py index 67f164bc0bb0..2d7e0acbf584 100644 --- a/examples/flax/summarization/run_summarization_flax.py +++ b/examples/flax/summarization/run_summarization_flax.py @@ -308,8 +308,13 @@ class DataTrainingArguments: ) def __post_init__(self): - if self.dataset_name is None and self.train_file is None and self.validation_file is None: - raise ValueError("Need either a dataset name or a training/validation file.") + if ( + self.dataset_name is None + and self.train_file is None + and self.validation_file is None + and self.test_file is None + ): + raise ValueError("Need either a dataset name or a training, validation, or test file.") else: if self.train_file is not None: extension = self.train_file.split(".")[-1] @@ -317,6 +322,9 @@ def __post_init__(self): if self.validation_file is not None: extension = self.validation_file.split(".")[-1] assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file." + if self.test_file is not None: + extension = self.test_file.split(".")[-1] + assert extension in ["csv", "json"], "`test_file` should be a csv or a json file." if self.val_max_target_length is None: self.val_max_target_length = self.max_target_length @@ -553,10 +561,16 @@ def main(): # Preprocessing the datasets. # We need to tokenize inputs and targets. if training_args.do_train: + if "train" not in dataset: + raise ValueError("--do_train requires a train dataset") column_names = dataset["train"].column_names elif training_args.do_eval: + if "validation" not in dataset: + raise ValueError("--do_eval requires a validation dataset") column_names = dataset["validation"].column_names elif training_args.do_predict: + if "test" not in dataset: + raise ValueError("--do_predict requires a test dataset") column_names = dataset["test"].column_names else: logger.info("There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.") @@ -620,8 +634,6 @@ def preprocess_function(examples): return model_inputs if training_args.do_train: - if "train" not in dataset: - raise ValueError("--do_train requires a train dataset") train_dataset = dataset["train"] if data_args.max_train_samples is not None: max_train_samples = min(len(train_dataset), data_args.max_train_samples) @@ -637,8 +649,6 @@ def preprocess_function(examples): if training_args.do_eval: max_target_length = data_args.val_max_target_length - if "validation" not in dataset: - raise ValueError("--do_eval requires a validation dataset") eval_dataset = dataset["validation"] if data_args.max_eval_samples is not None: max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) @@ -654,8 +664,6 @@ def preprocess_function(examples): if training_args.do_predict: max_target_length = data_args.val_max_target_length - if "test" not in dataset: - raise ValueError("--do_predict requires a test dataset") predict_dataset = dataset["test"] if data_args.max_predict_samples is not None: max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) diff --git a/examples/pytorch/summarization/run_summarization.py b/examples/pytorch/summarization/run_summarization.py index e5566f936328..587ff5b770ed 100755 --- a/examples/pytorch/summarization/run_summarization.py +++ b/examples/pytorch/summarization/run_summarization.py @@ -262,8 +262,13 @@ class DataTrainingArguments: ) def __post_init__(self): - if self.dataset_name is None and self.train_file is None and self.validation_file is None: - raise ValueError("Need either a dataset name or a training/validation file.") + if ( + self.dataset_name is None + and self.train_file is None + and self.validation_file is None + and self.test_file is None + ): + raise ValueError("Need either a dataset name or a training, validation, or test file.") else: if self.train_file is not None: extension = self.train_file.split(".")[-1] @@ -271,6 +276,9 @@ def __post_init__(self): if self.validation_file is not None: extension = self.validation_file.split(".")[-1] assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file." + if self.test_file is not None: + extension = self.test_file.split(".")[-1] + assert extension in ["csv", "json"], "`test_file` should be a csv or a json file." if self.val_max_target_length is None: self.val_max_target_length = self.max_target_length @@ -467,10 +475,16 @@ def main(): # Preprocessing the datasets. # We need to tokenize inputs and targets. if training_args.do_train: + if "train" not in raw_datasets: + raise ValueError("--do_train requires a train dataset") column_names = raw_datasets["train"].column_names elif training_args.do_eval: + if "validation" not in raw_datasets: + raise ValueError("--do_eval requires a validation dataset") column_names = raw_datasets["validation"].column_names elif training_args.do_predict: + if "test" not in raw_datasets: + raise ValueError("--do_predict requires a test dataset") column_names = raw_datasets["test"].column_names else: logger.info("There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.") @@ -546,8 +560,6 @@ def preprocess_function(examples): return model_inputs if training_args.do_train: - if "train" not in raw_datasets: - raise ValueError("--do_train requires a train dataset") train_dataset = raw_datasets["train"] if data_args.max_train_samples is not None: max_train_samples = min(len(train_dataset), data_args.max_train_samples) @@ -564,8 +576,6 @@ def preprocess_function(examples): if training_args.do_eval: max_target_length = data_args.val_max_target_length - if "validation" not in raw_datasets: - raise ValueError("--do_eval requires a validation dataset") eval_dataset = raw_datasets["validation"] if data_args.max_eval_samples is not None: max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) @@ -582,8 +592,6 @@ def preprocess_function(examples): if training_args.do_predict: max_target_length = data_args.val_max_target_length - if "test" not in raw_datasets: - raise ValueError("--do_predict requires a test dataset") predict_dataset = raw_datasets["test"] if data_args.max_predict_samples is not None: max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) From 4072d18c9e6a61fff427c8088ff5a06061707f61 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Wed, 22 Mar 2023 04:14:54 -0700 Subject: [PATCH 158/277] Fix position embeddings for GPT-J and CodeGen (#22069) * Revert "[GPT-J] add deprecation warning (#21869)" This reverts commit fb76994c41d1eaf09e50020cbd849d3bb686b6a3. * Fix position embeddings for GPT-J and CodeGen * Address review comments from @gante * Fix "Copied from" comment referencing wrong function * Fix copy/paste mistake * Fix training path * Hopefully make torch.fx happy * Move position_ids long cast * Revert "Hopefully make torch.fx happy" This reverts commit e41a6f4cad3ff441124c7457b19cfb630d4ca025. * Changes to help with torch.fx tracing * Linter fix * Correct position_ids tensor type hint * Work-around torch.fx tracing issue * Get the changes to work with torch.fx * Address review comment from @michaelbenayoun * Another small adjustment * Add explanatory comment; small code tidyup --- .../models/codegen/modeling_codegen.py | 78 ++++---- src/transformers/models/gptj/modeling_gptj.py | 176 +++++++++--------- src/transformers/utils/fx.py | 33 ++++ 3 files changed, 151 insertions(+), 136 deletions(-) diff --git a/src/transformers/models/codegen/modeling_codegen.py b/src/transformers/models/codegen/modeling_codegen.py index 0412b87b83a9..c095ed3e7bef 100644 --- a/src/transformers/models/codegen/modeling_codegen.py +++ b/src/transformers/models/codegen/modeling_codegen.py @@ -51,43 +51,26 @@ ] -# Copied from transformers.models.gptj.modeling_gptj.fixed_pos_embedding -def fixed_pos_embedding(x, seq_dim=1, seq_len=None): - dim = x.shape[-1] - if seq_len is None: - seq_len = x.shape[seq_dim] +# Copied from transformers.models.gptj.modeling_gptj.create_sinusoidal_positions +def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor: inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim)) - sinusoid_inp = ( - torch.einsum("i , j -> i j", torch.arange(seq_len, dtype=torch.float), inv_freq).to(x.device).float() - ) - return torch.sin(sinusoid_inp), torch.cos(sinusoid_inp) + sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float() + return torch.concat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1) # Copied from transformers.models.gptj.modeling_gptj.rotate_every_two -def rotate_every_two(x): +def rotate_every_two(x: torch.Tensor) -> torch.Tensor: x1 = x[:, :, :, ::2] x2 = x[:, :, :, 1::2] x = torch.stack((-x2, x1), dim=-1) return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)') -# Copied from transformers.models.gptj.modeling_gptj.duplicate_interleave -def duplicate_interleave(m): - """ - A simple version of `torch.repeat_interleave` for duplicating a matrix while interleaving the copy. - """ - dim0 = m.shape[0] - m = m.view(-1, 1) # flatten the matrix - m = m.repeat(1, 2) # repeat all elements into the 2nd dimension - m = m.view(dim0, -1) # reshape into a matrix, interleaving the copy - return m - - # Copied from transformers.models.gptj.modeling_gptj.apply_rotary_pos_emb -def apply_rotary_pos_emb(x, sincos, offset=0): - sin, cos = (duplicate_interleave(t)[None, offset : x.shape[1] + offset, None, :] for t in sincos) - # einsum notation for lambda t: repeat(t[offset:x.shape[1]+offset,:], "n d -> () n () (d j)", j=2) - return (x * cos) + (rotate_every_two(x) * sin) +def apply_rotary_pos_emb(tensor: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor: + sin = torch.repeat_interleave(sin[:, :, None, :], 2, 3) + cos = torch.repeat_interleave(cos[:, :, None, :], 2, 3) + return (tensor * cos) + (rotate_every_two(tensor) * sin) class CodeGenAttention(nn.Module): @@ -117,9 +100,9 @@ def __init__(self, config): self.qkv_proj = nn.Linear(self.embed_dim, self.embed_dim * 3, bias=False) self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False) - self.rotary_dim = None - if config.rotary_dim is not None: - self.rotary_dim = config.rotary_dim + self.rotary_dim = config.rotary_dim + pos_embd_dim = self.rotary_dim or self.embed_dim + self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim) def _split_heads(self, x, n_head, dim_head, mp_num): reshaped = x.reshape(x.shape[:-1] + (n_head // mp_num, dim_head)) @@ -183,8 +166,9 @@ def _attn( def forward( self, hidden_states: Optional[torch.FloatTensor], - attention_mask: Optional[torch.FloatTensor] = None, layer_past: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = False, output_attentions: Optional[bool] = False, @@ -205,12 +189,13 @@ def forward( value = self._split_heads(value, self.num_attention_heads, self.head_dim, mp_num=mp_num) value = value.permute(0, 2, 1, 3) - seq_len = key.shape[1] - offset = 0 + embed_positions = self.embed_positions + if embed_positions.device != position_ids.device: + embed_positions = embed_positions.to(position_ids.device) + self.embed_positions = embed_positions - if layer_past is not None: - offset = layer_past[0].shape[-2] - seq_len += offset + sincos = embed_positions[position_ids] + sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1) if self.rotary_dim is not None: k_rot = key[:, :, :, : self.rotary_dim] @@ -219,16 +204,14 @@ def forward( q_rot = query[:, :, :, : self.rotary_dim] q_pass = query[:, :, :, self.rotary_dim :] - sincos = fixed_pos_embedding(k_rot, 1, seq_len=seq_len) - k_rot = apply_rotary_pos_emb(k_rot, sincos, offset=offset) - q_rot = apply_rotary_pos_emb(q_rot, sincos, offset=offset) + k_rot = apply_rotary_pos_emb(k_rot, sin, cos) + q_rot = apply_rotary_pos_emb(q_rot, sin, cos) key = torch.cat([k_rot, k_pass], dim=-1) query = torch.cat([q_rot, q_pass], dim=-1) else: - sincos = fixed_pos_embedding(key, 1, seq_len=seq_len) - key = apply_rotary_pos_emb(key, sincos, offset=offset) - query = apply_rotary_pos_emb(query, sincos, offset=offset) + key = apply_rotary_pos_emb(key, sin, cos) + query = apply_rotary_pos_emb(query, sin, cos) key = key.permute(0, 2, 1, 3) query = query.permute(0, 2, 1, 3) @@ -292,6 +275,7 @@ def forward( hidden_states: Optional[torch.FloatTensor], layer_past: Optional[Tuple[torch.Tensor]] = None, attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = False, output_attentions: Optional[bool] = False, @@ -299,9 +283,10 @@ def forward( residual = hidden_states hidden_states = self.ln_1(hidden_states) attn_outputs = self.attn( - hidden_states, + hidden_states=hidden_states, layer_past=layer_past, attention_mask=attention_mask, + position_ids=position_ids, head_mask=head_mask, use_cache=use_cache, output_attentions=output_attentions, @@ -488,7 +473,7 @@ def forward( token_type_ids = token_type_ids.view(-1, input_shape[-1]) if position_ids is not None: - position_ids = position_ids.view(-1, input_shape[-1]) + position_ids = position_ids.view(-1, input_shape[-1]).long() if past_key_values is None: past_length = 0 @@ -568,13 +553,15 @@ def custom_forward(*inputs): hidden_states, None, attention_mask, + position_ids, head_mask[i], ) else: outputs = block( - hidden_states, + hidden_states=hidden_states, layer_past=layer_past, attention_mask=attention_mask, + position_ids=position_ids, head_mask=head_mask[i], use_cache=use_cache, output_attentions=output_attentions, @@ -645,8 +632,7 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwarg position_ids.masked_fill_(attention_mask == 0, 1) if past_key_values: position_ids = position_ids[:, -1].unsqueeze(-1) - else: - position_ids = None + return { "input_ids": input_ids, "past_key_values": past_key_values, diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index 959aa48dbd32..9f2b5642efa5 100755 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -30,7 +30,13 @@ SequenceClassifierOutputWithPast, ) from ...modeling_utils import PreTrainedModel -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging +from ...utils import ( + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_torch_fx_proxy, + logging, +) from ...utils.model_parallel_utils import assert_device_map, get_device_map from .configuration_gptj import GPTJConfig @@ -48,39 +54,28 @@ ] -def fixed_pos_embedding(x, seq_dim=1, seq_len=None): - dim = x.shape[-1] - if seq_len is None: - seq_len = x.shape[seq_dim] +def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor: inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim)) - sinusoid_inp = ( - torch.einsum("i , j -> i j", torch.arange(seq_len, dtype=torch.float), inv_freq).to(x.device).float() - ) - return torch.sin(sinusoid_inp), torch.cos(sinusoid_inp) + sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float() + return torch.concat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1) + + +@torch.fx.wrap +def get_embed_positions(embed_positions, position_ids): + return embed_positions.to(position_ids.device).repeat(position_ids.shape[0], 1, 1) -def rotate_every_two(x): +def rotate_every_two(x: torch.Tensor) -> torch.Tensor: x1 = x[:, :, :, ::2] x2 = x[:, :, :, 1::2] x = torch.stack((-x2, x1), dim=-1) return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)') -def duplicate_interleave(m): - """ - A simple version of `torch.repeat_interleave` for duplicating a matrix while interleaving the copy. - """ - dim0 = m.shape[0] - m = m.view(-1, 1) # flatten the matrix - m = m.repeat(1, 2) # repeat all elements into the 2nd dimension - m = m.view(dim0, -1) # reshape into a matrix, interleaving the copy - return m - - -def apply_rotary_pos_emb(x, sincos, offset=0): - sin, cos = (duplicate_interleave(t)[None, offset : x.shape[1] + offset, None, :] for t in sincos) - # einsum notation for lambda t: repeat(t[offset:x.shape[1]+offset,:], "n d -> () n () (d j)", j=2) - return (x * cos) + (rotate_every_two(x) * sin) +def apply_rotary_pos_emb(tensor: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor: + sin = torch.repeat_interleave(sin[:, :, None, :], 2, 3) + cos = torch.repeat_interleave(cos[:, :, None, :], 2, 3) + return (tensor * cos) + (rotate_every_two(tensor) * sin) class GPTJAttention(nn.Module): @@ -113,9 +108,9 @@ def __init__(self, config): self.v_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False) self.q_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False) self.out_proj = nn.Linear(self.embed_dim, self.embed_dim, bias=False) - self.rotary_dim = None - if config.rotary_dim is not None: - self.rotary_dim = config.rotary_dim + self.rotary_dim = config.rotary_dim + pos_embd_dim = self.rotary_dim or self.embed_dim + self.embed_positions = create_sinusoidal_positions(max_positions, pos_embd_dim) def _split_heads(self, tensor, num_attention_heads, attn_head_size, rotary): """ @@ -187,11 +182,19 @@ def _attn( return attn_output, attn_weights + def _get_embed_positions(self, position_ids): + embed_positions = self.embed_positions + if embed_positions.device != position_ids.device: + embed_positions = embed_positions.to(position_ids.device) + self.embed_positions = embed_positions + return embed_positions.repeat(position_ids.shape[0], 1, 1) + def forward( self, hidden_states: Optional[torch.FloatTensor], - attention_mask: Optional[torch.FloatTensor] = None, layer_past: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = False, output_attentions: Optional[bool] = False, @@ -207,12 +210,16 @@ def forward( key = self._split_heads(key, self.num_attention_heads, self.head_dim, True) value = self._split_heads(value, self.num_attention_heads, self.head_dim, False) - seq_len = key.shape[1] - offset = 0 + if is_torch_fx_proxy(position_ids): + # The logic to conditionally copy to GPU could not be traced, so we do this + # every time in the torch.fx case + embed_positions = get_embed_positions(self.embed_positions, position_ids) + else: + embed_positions = self._get_embed_positions(position_ids) - if layer_past is not None: - offset = layer_past[0].shape[-2] - seq_len += offset + repeated_position_ids = position_ids.unsqueeze(-1).repeat(1, 1, embed_positions.shape[-1]) + sincos = torch.gather(embed_positions, 1, repeated_position_ids) + sin, cos = torch.split(sincos, sincos.shape[-1] // 2, dim=-1) if self.rotary_dim is not None: k_rot = key[:, :, :, : self.rotary_dim] @@ -221,16 +228,14 @@ def forward( q_rot = query[:, :, :, : self.rotary_dim] q_pass = query[:, :, :, self.rotary_dim :] - sincos = fixed_pos_embedding(k_rot, 1, seq_len=seq_len) - k_rot = apply_rotary_pos_emb(k_rot, sincos, offset=offset) - q_rot = apply_rotary_pos_emb(q_rot, sincos, offset=offset) + k_rot = apply_rotary_pos_emb(k_rot, sin, cos) + q_rot = apply_rotary_pos_emb(q_rot, sin, cos) key = torch.cat([k_rot, k_pass], dim=-1) query = torch.cat([q_rot, q_pass], dim=-1) else: - sincos = fixed_pos_embedding(key, 1, seq_len=seq_len) - key = apply_rotary_pos_emb(key, sincos, offset=offset) - query = apply_rotary_pos_emb(query, sincos, offset=offset) + key = apply_rotary_pos_emb(key, sin, cos) + query = apply_rotary_pos_emb(query, sin, cos) key = key.permute(0, 2, 1, 3) query = query.permute(0, 2, 1, 3) @@ -292,6 +297,7 @@ def forward( hidden_states: Optional[torch.FloatTensor], layer_past: Optional[Tuple[torch.Tensor]] = None, attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = False, output_attentions: Optional[bool] = False, @@ -299,9 +305,10 @@ def forward( residual = hidden_states hidden_states = self.ln_1(hidden_states) attn_outputs = self.attn( - hidden_states, + hidden_states=hidden_states, layer_past=layer_past, attention_mask=attention_mask, + position_ids=position_ids, head_mask=head_mask, use_cache=use_cache, output_attentions=output_attentions, @@ -391,6 +398,11 @@ def _set_gradient_checkpointing(self, module, value=False): - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) + position_ids (`torch.LongTensor` of shape `({0})`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: @@ -544,24 +556,14 @@ def forward( past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, attention_mask: Optional[torch.FloatTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, inputs_embeds: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - **deprecated_arguments, ) -> Union[Tuple, BaseModelOutputWithPast]: - if deprecated_arguments.pop("position_ids", False) is not False: - # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` - warnings.warn( - "`position_ids` have no functionality in GPT-J and will be removed in v5.0.0. You can safely ignore" - " passing `position_ids`.", - FutureWarning, - ) - if len(deprecated_arguments) > 0: - raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -581,11 +583,23 @@ def forward( else: raise ValueError("You have to specify either input_ids or inputs_embeds") + device = input_ids.device if input_ids is not None else inputs_embeds.device + if token_type_ids is not None: token_type_ids = token_type_ids.view(-1, input_shape[-1]) + if position_ids is not None: + position_ids = position_ids.view(-1, input_shape[-1]).long() + if past_key_values is None: + past_length = 0 past_key_values = tuple([None] * len(self.h)) + else: + past_length = past_key_values[0][0].size(-2) + + if position_ids is None: + position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1]) # Attention mask. if attention_mask is not None: @@ -665,13 +679,15 @@ def custom_forward(*inputs): hidden_states, None, attention_mask, + position_ids, head_mask[i], ) else: outputs = block( - hidden_states, + hidden_states=hidden_states, layer_past=layer_past, attention_mask=attention_mask, + position_ids=position_ids, head_mask=head_mask[i], use_cache=use_cache, output_attentions=output_attentions, @@ -766,9 +782,7 @@ def get_output_embeddings(self): def set_output_embeddings(self, new_embeddings): self.lm_head = new_embeddings - def prepare_inputs_for_generation( - self, input_ids, past_key_values=None, inputs_embeds=None, use_cache=None, **kwargs - ): + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs): token_type_ids = kwargs.get("token_type_ids", None) # only last token for inputs_ids if past is defined in kwargs if past_key_values: @@ -777,6 +791,14 @@ def prepare_inputs_for_generation( token_type_ids = token_type_ids[:, -1].unsqueeze(-1) attention_mask = kwargs.get("attention_mask", None) + position_ids = kwargs.get("position_ids", None) + + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) # if `inputs_embeds` are passed, we only want to use them in the 1st generation step if inputs_embeds is not None and past_key_values is None: @@ -787,7 +809,8 @@ def prepare_inputs_for_generation( model_inputs.update( { "past_key_values": past_key_values, - "use_cache": use_cache, + "use_cache": kwargs.get("use_cache"), + "position_ids": position_ids, "attention_mask": attention_mask, "token_type_ids": token_type_ids, } @@ -808,6 +831,7 @@ def forward( past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, attention_mask: Optional[torch.FloatTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, @@ -815,7 +839,6 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - **deprecated_arguments, ) -> Union[Tuple, CausalLMOutputWithPast]: r""" labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -823,16 +846,6 @@ def forward( `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` """ - if deprecated_arguments.pop("position_ids", False) is not False: - # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` - warnings.warn( - "`position_ids` have no functionality in GPT-J and will be removed in v5.0.0. You can safely ignore" - " passing `position_ids`.", - FutureWarning, - ) - if len(deprecated_arguments) > 0: - raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") - return_dict = return_dict if return_dict is not None else self.config.use_return_dict transformer_outputs = self.transformer( @@ -840,6 +853,7 @@ def forward( past_key_values=past_key_values, attention_mask=attention_mask, token_type_ids=token_type_ids, + position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, use_cache=use_cache, @@ -941,6 +955,7 @@ def forward( past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, attention_mask: Optional[torch.FloatTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, @@ -948,7 +963,6 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - **deprecated_arguments, ) -> Union[Tuple, SequenceClassifierOutputWithPast]: r""" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): @@ -956,16 +970,6 @@ def forward( config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). """ - if deprecated_arguments.pop("position_ids", False) is not False: - # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` - warnings.warn( - "`position_ids` have no functionality in GPT-J and will be removed in v5.0.0. You can safely ignore" - " passing `position_ids`.", - FutureWarning, - ) - if len(deprecated_arguments) > 0: - raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") - return_dict = return_dict if return_dict is not None else self.config.use_return_dict transformer_outputs = self.transformer( @@ -973,6 +977,7 @@ def forward( past_key_values=past_key_values, attention_mask=attention_mask, token_type_ids=token_type_ids, + position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, use_cache=use_cache, @@ -1074,6 +1079,7 @@ def forward( input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.FloatTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, inputs_embeds: Optional[torch.FloatTensor] = None, start_positions: Optional[torch.LongTensor] = None, @@ -1081,7 +1087,6 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - **deprecated_arguments, ) -> Union[Tuple, QuestionAnsweringModelOutput]: r""" start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): @@ -1093,22 +1098,13 @@ def forward( Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. """ - if deprecated_arguments.pop("position_ids", False) is not False: - # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` - warnings.warn( - "`position_ids` have no functionality in GPT-J and will be removed in v5.0.0. You can safely ignore" - " passing `position_ids`.", - FutureWarning, - ) - if len(deprecated_arguments) > 0: - raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") - return_dict = return_dict if return_dict is not None else self.config.use_return_dict outputs = self.transformer( input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, + position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, output_attentions=output_attentions, diff --git a/src/transformers/utils/fx.py b/src/transformers/utils/fx.py index 9da544cb45e9..350d9e91245a 100755 --- a/src/transformers/utils/fx.py +++ b/src/transformers/utils/fx.py @@ -363,6 +363,26 @@ def torch_tensor_repeat(self, *sizes): return torch.empty(shape, device="meta") +def torch_repeat_interleave(*args, dim=None, output_size=None): + num_args = len(args) + if num_args == 1: + shape = [output_size if output_size is not None else args[0].sum()] + else: + shape = list(args[0].shape) + if dim is None: + if num_args > 2: + dim = args[2] + else: + shape = [sum(shape)] + dim = 0 + repeats = args[1] + if isinstance(repeats, int) or torch.numel(repeats) == 1: + shape[dim] *= int(repeats) + else: + shape[dim] = output_size if output_size is not None else repeats.sum() + return torch.empty(*shape, device="meta") + + def torch_index_select(input, dim, index, *, out=None): shape = list(input.shape) shape[dim] = len(index) @@ -373,6 +393,16 @@ def torch_tensor_index_select(self, dim, index): return torch_index_select(self, dim, index) +def torch_gather(input, dim, index, *, sparse_grad=False, out=None): + shape = list(input.shape) + shape[dim] = index.shape[dim] + return torch.empty(*shape, device="meta") + + +def torch_tensor_gather(self, dim, index): + return torch_gather(self, dim, index) + + def torch_roll(input, shifts, dims=None): return input @@ -539,11 +569,14 @@ def to_concrete(t): torch.Tensor.baddbmm: torch_tensor_baddbmm, torch.einsum: torch_einsum, torch.Tensor.repeat: torch_tensor_repeat, + torch.repeat_interleave: torch_repeat_interleave, torch.roll: torch_roll, torch.flip: torch_flip, torch.Tensor.flip: torch_tensor_flip, torch.index_select: torch_index_select, torch.Tensor.index_select: torch_tensor_index_select, + torch.gather: torch_gather, + torch.Tensor.gather: torch_tensor_gather, torch.nn.Conv1d: torch_nn_conv1d, torch.nn.Conv2d: torch_nn_conv2d, torch.squeeze: torch_squeeze, From 963539e3a13de1c80f28f76f86b2012f2b35dd64 Mon Sep 17 00:00:00 2001 From: silentghoul-spec <58596410+silentghoul-spec@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:37:49 +0530 Subject: [PATCH 159/277] Fixed bug to calculate correct xpath_sub_list in MarkupLMTokenizer (#22302) Fixed bug to calculate correct xpath_sub_list in MarkupLMTokenizer. Earlier xpath_sub_list was same as xpath_tags_list Co-authored-by: dusejat --- .../models/markuplm/tokenization_markuplm.py | 2 +- .../markuplm/tokenization_markuplm_fast.py | 2 +- .../markuplm/test_tokenization_markuplm.py | 18 ++++++------------ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/transformers/models/markuplm/tokenization_markuplm.py b/src/transformers/models/markuplm/tokenization_markuplm.py index f4eb0af4d9b7..9d4386028646 100644 --- a/src/transformers/models/markuplm/tokenization_markuplm.py +++ b/src/transformers/models/markuplm/tokenization_markuplm.py @@ -301,7 +301,7 @@ def get_xpath_seq(self, xpath): xpath_subs_list.append(min(self.max_width, sub)) xpath_tags_list = xpath_tags_list[: self.max_depth] - xpath_subs_list = xpath_tags_list[: self.max_depth] + xpath_subs_list = xpath_subs_list[: self.max_depth] xpath_tags_list += [self.pad_tag_id] * (self.max_depth - len(xpath_tags_list)) xpath_subs_list += [self.pad_width] * (self.max_depth - len(xpath_subs_list)) diff --git a/src/transformers/models/markuplm/tokenization_markuplm_fast.py b/src/transformers/models/markuplm/tokenization_markuplm_fast.py index 8c1cb73afdaa..0010c21cdce5 100644 --- a/src/transformers/models/markuplm/tokenization_markuplm_fast.py +++ b/src/transformers/models/markuplm/tokenization_markuplm_fast.py @@ -275,7 +275,7 @@ def get_xpath_seq(self, xpath): xpath_subs_list.append(min(self.max_width, sub)) xpath_tags_list = xpath_tags_list[: self.max_depth] - xpath_subs_list = xpath_tags_list[: self.max_depth] + xpath_subs_list = xpath_subs_list[: self.max_depth] xpath_tags_list += [self.pad_tag_id] * (self.max_depth - len(xpath_tags_list)) xpath_subs_list += [self.pad_width] * (self.max_depth - len(xpath_subs_list)) diff --git a/tests/models/markuplm/test_tokenization_markuplm.py b/tests/models/markuplm/test_tokenization_markuplm.py index e59934e4d086..533a3429a8dc 100644 --- a/tests/models/markuplm/test_tokenization_markuplm.py +++ b/tests/models/markuplm/test_tokenization_markuplm.py @@ -2186,8 +2186,7 @@ def test_markuplm_integration_test(self): nodes, xpaths = self.get_nodes_and_xpaths() # fmt: off - expected_results = {'input_ids': [0, 42891, 8331, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'xpath_tags_seq': [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], 'xpath_subs_seq': [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} - # fmt: on + expected_results = {'input_ids': [0, 42891, 8331, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'xpath_tags_seq': [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], 'xpath_subs_seq': [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} # fmt: on encoding_p = tokenizer_p(nodes, xpaths=xpaths, padding="max_length", max_length=20) encoding_r = tokenizer_r(nodes, xpaths=xpaths, padding="max_length", max_length=20) @@ -2198,7 +2197,7 @@ def test_markuplm_integration_test(self): nodes, xpaths = self.get_nodes_and_xpaths_batch() # fmt: off - expected_results = {'input_ids': [[0, 42891, 232, 12364, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 42891, 127, 766, 16, 22401, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'xpath_tags_seq': [[[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]]], 'xpath_subs_seq': [[[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]} + expected_results = {'input_ids': [[0, 42891, 232, 12364, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 42891, 127, 766, 16, 22401, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'xpath_tags_seq': [[[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]]], 'xpath_subs_seq': [[[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]} # fmt: on encoding_p = tokenizer_p(nodes, xpaths=xpaths, padding="max_length", max_length=20) @@ -2211,7 +2210,7 @@ def test_markuplm_integration_test(self): node_labels = [1, 2, 3] # fmt: off - expected_results = {'input_ids': [0, 42891, 8331, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'xpath_tags_seq': [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], 'xpath_subs_seq': [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'labels': [-100, 1, 2, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100], 'attention_mask': [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} + expected_results = {'input_ids': [0, 42891, 8331, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'xpath_tags_seq': [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], 'xpath_subs_seq': [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'labels': [-100, 1, 2, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100], 'attention_mask': [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} # fmt: on encoding_p = tokenizer_p(nodes, xpaths=xpaths, node_labels=node_labels, padding="max_length", max_length=20) @@ -2224,8 +2223,7 @@ def test_markuplm_integration_test(self): node_labels = [[1, 2, 3], [2, 46, 17, 22, 3]] # fmt: off - expected_results = {'input_ids': [[0, 42891, 232, 12364, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 42891, 127, 766, 16, 22401, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'xpath_tags_seq': [[[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]]], 'xpath_subs_seq': [[[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'labels': [[-100, 1, -100, 2, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100], [-100, 2, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100]], 'attention_mask': [[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]} - # fmt: on + expected_results = {'input_ids': [[0, 42891, 232, 12364, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 42891, 127, 766, 16, 22401, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'xpath_tags_seq': [[[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]]], 'xpath_subs_seq': [[[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'labels': [[-100, 1, -100, 2, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100], [-100, 2, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100]], 'attention_mask': [[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]} # fmt: on encoding_p = tokenizer_p(nodes, xpaths=xpaths, node_labels=node_labels, padding="max_length", max_length=20) encoding_r = tokenizer_r(nodes, xpaths=xpaths, node_labels=node_labels, padding="max_length", max_length=20) @@ -2236,8 +2234,7 @@ def test_markuplm_integration_test(self): question, nodes, xpaths = self.get_question_nodes_and_xpaths() # fmt: off - expected_results = {'input_ids': [0, 12196, 18, 39, 766, 116, 2, 42891, 232, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], - 'xpath_tags_seq': [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], 'xpath_subs_seq': [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} + expected_results = {'input_ids': [0, 12196, 18, 39, 766, 116, 2, 42891, 232, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'xpath_tags_seq': [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], 'xpath_subs_seq': [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} # fmt: on encoding_p = tokenizer_p(question, nodes, xpaths, padding="max_length", max_length=20) @@ -2249,10 +2246,7 @@ def test_markuplm_integration_test(self): questions, nodes, xpaths = self.get_question_nodes_and_xpaths_batch() # fmt: off - expected_results = {'input_ids': [[0, 12196, 18, 39, 766, 116, 2, 42891, 232, 12364, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 9178, 16, 37, 373, 116, 2, 42891, 127, 766, 16, 22401, 2, 1, 1, 1, 1, 1, 1, 1]], - 'xpath_tags_seq': [[[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]]], - 'xpath_subs_seq': [[[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [109, 25, 50, 120, 50, 178, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]]], - 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]]} + expected_results = {'input_ids': [[0, 12196, 18, 39, 766, 116, 2, 42891, 232, 12364, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 9178, 16, 37, 373, 116, 2, 42891, 127, 766, 16, 22401, 2, 1, 1, 1, 1, 1, 1, 1]], 'xpath_tags_seq': [[[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]], [[216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [109, 25, 50, 120, 50, 178, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216], [216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, 216]]], 'xpath_subs_seq': [[[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 1, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]], [[1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [0, 0, 0, 2, 0, 0, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001], [1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001, 1001]]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]]} # fmt: on encoding_p = tokenizer_p(questions, nodes, xpaths, padding="max_length", max_length=20) From eb6409db9d53797c774d6373632e70c13646d841 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 22 Mar 2023 09:22:07 -0400 Subject: [PATCH 160/277] Enforce `max_memory` for device_map strategies (#22311) Enforce for device_map strategies --- src/transformers/modeling_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 94bbe4a9d5f9..6245a0221158 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2571,6 +2571,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P model, dtype=torch_dtype, low_zero=(device_map == "balanced_low_0"), + max_memory=max_memory, **kwargs, ) kwargs["max_memory"] = max_memory From e0b2ba23f0c8b0fbd290c385bc597f14ede749d4 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 22 Mar 2023 15:00:20 +0000 Subject: [PATCH 161/277] Generate: Export TF generate with a TF tokenizer (#22310) * Export TF generate with a TF tokenizer * remove unused lines --- src/transformers/generation/tf_utils.py | 84 +++++++++---------------- tests/generation/test_tf_utils.py | 39 +++++++++++- 2 files changed, 68 insertions(+), 55 deletions(-) diff --git a/src/transformers/generation/tf_utils.py b/src/transformers/generation/tf_utils.py index 4a9140f8853d..749c07d547c7 100644 --- a/src/transformers/generation/tf_utils.py +++ b/src/transformers/generation/tf_utils.py @@ -1725,14 +1725,13 @@ def greedy_search_body_fn(generated, finished_sequences, cur_len, model_kwargs): # 2-to-n generation steps can then be run in autoregressive fashion # only in case 1st generation step does NOT yield EOS token though - if greedy_search_cond_fn(generated, finished_sequences, cur_len, model_kwargs): - maximum_iterations = max_length - cur_len - generated, _, cur_len, _ = tf.while_loop( - greedy_search_cond_fn, - greedy_search_body_fn, - (generated, finished_sequences, cur_len, model_kwargs), - maximum_iterations=maximum_iterations, - ) + maximum_iterations = max_length - cur_len + generated, _, cur_len, _ = tf.while_loop( + greedy_search_cond_fn, + greedy_search_body_fn, + (generated, finished_sequences, cur_len, model_kwargs), + maximum_iterations=maximum_iterations, + ) # 6. prepare outputs if not use_xla: @@ -2016,14 +2015,13 @@ def sample_body_fn(generated, finished_sequences, cur_len, model_kwargs): # 2-to-n generation steps can then be run in autoregressive fashion # only in case 1st generation step does NOT yield EOS token though - if sample_cond_fn(generated, finished_sequences, cur_len, model_kwargs): - maximum_iterations = max_length - cur_len - generated, _, cur_len, _ = tf.while_loop( - sample_cond_fn, - sample_body_fn, - (generated, finished_sequences, cur_len, model_kwargs), - maximum_iterations=maximum_iterations, - ) + maximum_iterations = max_length - cur_len + generated, _, cur_len, _ = tf.while_loop( + sample_cond_fn, + sample_body_fn, + (generated, finished_sequences, cur_len, model_kwargs), + maximum_iterations=maximum_iterations, + ) # 6. prepare outputs if not use_xla: @@ -2565,7 +2563,8 @@ def beam_search_body_fn( # 2-to-n generation steps can then be run in autoregressive fashion (only in case 1st generation step does # NOT yield EOS token though) - if beam_search_cond_fn( + maximum_iterations = max_length - cur_len + ( cur_len, running_sequences, running_scores, @@ -2574,9 +2573,10 @@ def beam_search_body_fn( scores, beam_indices, is_sent_finished, - model_kwargs, - ): - maximum_iterations = max_length - cur_len + _, + ) = tf.while_loop( + beam_search_cond_fn, + beam_search_body_fn, ( cur_len, running_sequences, @@ -2586,23 +2586,10 @@ def beam_search_body_fn( scores, beam_indices, is_sent_finished, - _, - ) = tf.while_loop( - beam_search_cond_fn, - beam_search_body_fn, - ( - cur_len, - running_sequences, - running_scores, - running_beam_indices, - sequences, - scores, - beam_indices, - is_sent_finished, - model_kwargs, - ), - maximum_iterations=maximum_iterations, - ) + model_kwargs, + ), + maximum_iterations=maximum_iterations, + ) # 6. prepare outputs # Account for the edge-case where there are no finished sequences for a particular batch item. If so, return @@ -3019,22 +3006,13 @@ def contrastive_search_body_fn( # 2-to-n generation steps can then be run in autoregressive fashion # only in case 1st generation step does NOT yield EOS token though - if contrastive_search_cond_fn( - generated, finished_sequences, cur_len, model_kwargs, next_step_cached_variables - ): - maximum_iterations = max_length - cur_len - ( - generated, - _, - cur_len, - _, - _, - ) = tf.while_loop( - contrastive_search_cond_fn, - contrastive_search_body_fn, - (generated, finished_sequences, cur_len, model_kwargs, next_step_cached_variables), - maximum_iterations=maximum_iterations, - ) + maximum_iterations = max_length - cur_len + generated, _, cur_len, _, _ = tf.while_loop( + contrastive_search_cond_fn, + contrastive_search_body_fn, + (generated, finished_sequences, cur_len, model_kwargs, next_step_cached_variables), + maximum_iterations=maximum_iterations, + ) # 6. prepare outputs if not use_xla: diff --git a/tests/generation/test_tf_utils.py b/tests/generation/test_tf_utils.py index cab4512bec6e..6fdad1ef6365 100644 --- a/tests/generation/test_tf_utils.py +++ b/tests/generation/test_tf_utils.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import tempfile import unittest import numpy as np +from huggingface_hub import hf_hub_download -from transformers import is_tf_available -from transformers.testing_utils import require_tf, slow +from transformers import is_tensorflow_text_available, is_tf_available +from transformers.testing_utils import require_tensorflow_text, require_tf, slow from ..test_modeling_tf_common import floats_tensor from .test_framework_agnostic import GenerationIntegrationTestsMixin @@ -40,6 +42,9 @@ tf_top_k_top_p_filtering, ) +if is_tensorflow_text_available(): + import tensorflow_text as text + @require_tf class UtilsFunctionsTest(unittest.TestCase): @@ -239,6 +244,36 @@ def serving(self, input_ids, attention_mask): tf_model_outputs = test_model.generate(**inputs, max_new_tokens=max_new_tokens) tf.debugging.assert_equal(tf_func_outputs, tf_model_outputs) + @slow + @require_tensorflow_text + def test_generate_tf_function_export_with_tf_tokenizer(self): + # TF-only test: tf.saved_model export + with tempfile.TemporaryDirectory() as tmp_dir: + # file needed to load the TF tokenizer + hf_hub_download(repo_id="google/flan-t5-small", filename="spiece.model", local_dir=tmp_dir) + + class CompleteSentenceTransformer(tf.keras.layers.Layer): + def __init__(self): + super().__init__() + self.tokenizer = text.SentencepieceTokenizer( + model=tf.io.gfile.GFile(os.path.join(tmp_dir, "spiece.model"), "rb").read() + ) + self.model = TFAutoModelForSeq2SeqLM.from_pretrained("hf-internal-testing/tiny-random-t5") + + def call(self, inputs, *args, **kwargs): + tokens = self.tokenizer.tokenize(inputs) + input_ids, attention_mask = text.pad_model_inputs( + tokens, max_seq_length=64, pad_value=self.model.config.pad_token_id + ) + outputs = self.model.generate(input_ids=input_ids, attention_mask=attention_mask) + return self.tokenizer.detokenize(outputs) + + complete_model = CompleteSentenceTransformer() + inputs = tf.keras.layers.Input(shape=(1,), dtype=tf.string, name="inputs") + outputs = complete_model(inputs) + keras_model = tf.keras.Model(inputs, outputs) + keras_model.save(tmp_dir) + def test_eos_token_id_int_and_list_top_k_top_sampling(self): # Has PT equivalent: this test relies on random sampling generation_kwargs = { From 88ce6d5947945b721de304dd43971cd22f6c1c12 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Wed, 22 Mar 2023 15:20:48 +0000 Subject: [PATCH 162/277] Beef up Llama tests (#22314) * tmp commit * beef up llama tests --- tests/generation/test_utils.py | 2 +- tests/models/llama/test_modeling_llama.py | 31 ++++++++++------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index de319c2b0004..a269b68d0020 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -1463,10 +1463,10 @@ def test_generate_with_head_masking(self): attention_names = ["encoder_attentions", "decoder_attentions", "cross_attentions"] for model_class in self.all_generative_model_classes: config, input_ids, attention_mask, max_length = self._get_input_ids_and_config() - model = model_class(config).to(torch_device) # We want to test only encoder-decoder models if not config.is_encoder_decoder: continue + model = model_class(config).to(torch_device) head_masking = { "head_mask": torch.zeros(config.encoder_layers, config.encoder_attention_heads, device=torch_device), diff --git a/tests/models/llama/test_modeling_llama.py b/tests/models/llama/test_modeling_llama.py index dea92d5111fd..113f74d30976 100644 --- a/tests/models/llama/test_modeling_llama.py +++ b/tests/models/llama/test_modeling_llama.py @@ -20,8 +20,10 @@ from transformers import LlamaConfig, is_torch_available from transformers.testing_utils import require_torch, torch_device +from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask +from ...test_pipeline_mixin import PipelineTesterMixin if is_torch_available(): @@ -254,10 +256,21 @@ def prepare_config_and_inputs_for_common(self): @require_torch -class LlamaModelTest(ModelTesterMixin, unittest.TestCase): +class LlamaModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = (LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else () all_generative_model_classes = (LlamaForCausalLM,) if is_torch_available() else () + pipeline_model_mapping = ( + { + "feature-extraction": LlamaModel, + "text-classification": LlamaForSequenceClassification, + "text-generation": LlamaForCausalLM, + "zero-shot": LlamaForSequenceClassification, + } + if is_torch_available() + else {} + ) test_headmasking = False + test_pruning = False def setUp(self): self.model_tester = LlamaModelTester(self) @@ -316,22 +329,6 @@ def test_llama_sequence_classification_model_for_multi_label(self): result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) - @unittest.skip("LLaMA does not support head pruning.") - def test_head_pruning(self): - pass - - @unittest.skip("LLaMA does not support head pruning.") - def test_head_pruning_integration(self): - pass - - @unittest.skip("LLaMA does not support head pruning.") - def test_head_pruning_save_load_from_config_init(self): - pass - - @unittest.skip("LLaMA does not support head pruning.") - def test_head_pruning_save_load_from_pretrained(self): - pass - @unittest.skip("LLaMA buffers include complex numbers, which breaks this test") def test_save_load_fast_init_from_base(self): pass From f9d07cc11fa054462e2146d7432403e566f4d95f Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 22 Mar 2023 16:53:52 +0100 Subject: [PATCH 163/277] Add Pix2Struct (#21400) * v1 all keys match * clean up * forward pass ok * add correct image transform * generate works, logits matching * clean up * more refactor * revert * revert * clean up * clean ups * clean up * refactor * refactor * fix doc * fix tokenizer test * fix toctree * revert toctree * oops * few fixes * replace to `pixel_embeds` * make fixup * test processing & feat extractor * fix some tests * more fixes * make fixup * clean up * more clean up * add a single slow test * fix test * make fixup * fix * fix authors * fix toctree * update docs * add docstring * revert change * Update src/transformers/models/pix2struct/__init__.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * fix tokenizer * fix processor test * fix test * make fixup * refactor * fix config * Update src/transformers/models/pix2struct/image_processing_pix2struct.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * format * fix * Update src/transformers/models/pix2struct/image_processing_pix2struct.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * make fixup * add docstring * fix issues * fix * fix * fix * add slow test * fix * fix * fix batched issue * fix training issues * fix ci test * fix slow test * fix conversion script * remove unneeded classes * fix slow test * fix require backends * fix masked fill * revert * fix softmax * add large models support * fix conditional generation * few fixes * add instructions * rm unneeded file * Update src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py * fix ci test * fix ci test really * Apply suggestions from code review Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * fix nit * fix nits * fix image processors nits * docstring * clean up * fix nit * fix tests * docstring nit * fix reshape * Update src/transformers/models/pix2struct/image_processing_pix2struct.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> * fix nit * fix repetition * refactor processor * make patch size consistent * refactor forward * fix docstring * fix max_patches issue * update docstirng * update docstring * fix coped from * add skip reasons * few fixes * Update src/transformers/models/pix2struct/image_processing_pix2struct.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> * format * fix doctests * refactor and fix * fix doc build issue * fix processor test * small fix conversion script * replace correct weights * make fixup * fix some issues * Apply suggestions from code review Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * revert config and fixes * Update src/transformers/models/pix2struct/image_processing_pix2struct.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * more details * fixes * fix processor * fix processor test * fix * Apply suggestions from code review Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * make fixup * fix processor * Update src/transformers/models/pix2struct/modeling_pix2struct.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * add copied * make fixup * fix copies * update docstring * refactor * fix docstring * fix conversion script * fix vqa issue * replace to `flattened_patches` * nit * fix numpy issue * fix image processors * add batched vqa support * fix vqa conversion * make fixup * fix conversion script * Apply suggestions from code review Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * make fixup * add correct docstring * update docstring * fix module level + channel dim * use `make_list_of_images` * refactor * correct docstring * fix authors * remove `data_format` * add header text test * Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * make fixup * add checkpoints --------- Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- README.md | 1 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/en/_toctree.yml | 2 + docs/source/en/index.mdx | 2 + docs/source/en/model_doc/pix2struct.mdx | 73 + src/transformers/__init__.py | 32 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + .../models/auto/image_processing_auto.py | 1 + .../models/auto/processing_auto.py | 1 + .../models/auto/tokenization_auto.py | 1 + .../models/pix2struct/__init__.py | 86 + .../pix2struct/configuration_pix2struct.py | 412 ++++ ...nvert_pix2struct_original_pytorch_to_hf.py | 155 ++ .../pix2struct/image_processing_pix2struct.py | 418 ++++ .../models/pix2struct/modeling_pix2struct.py | 1823 +++++++++++++++++ .../pix2struct/processing_pix2struct.py | 162 ++ src/transformers/utils/dummy_pt_objects.py | 31 + .../utils/dummy_vision_objects.py | 7 + tests/models/pix2struct/__init__.py | 0 .../test_image_processing_pix2struct.py | 289 +++ .../pix2struct/test_modeling_pix2struct.py | 711 +++++++ .../pix2struct/test_processor_pix2struct.py | 192 ++ utils/check_repo.py | 3 + 29 files changed, 4412 insertions(+) create mode 100644 docs/source/en/model_doc/pix2struct.mdx create mode 100644 src/transformers/models/pix2struct/__init__.py create mode 100644 src/transformers/models/pix2struct/configuration_pix2struct.py create mode 100644 src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py create mode 100644 src/transformers/models/pix2struct/image_processing_pix2struct.py create mode 100644 src/transformers/models/pix2struct/modeling_pix2struct.py create mode 100644 src/transformers/models/pix2struct/processing_pix2struct.py create mode 100644 tests/models/pix2struct/__init__.py create mode 100644 tests/models/pix2struct/test_image_processing_pix2struct.py create mode 100644 tests/models/pix2struct/test_modeling_pix2struct.py create mode 100644 tests/models/pix2struct/test_processor_pix2struct.py diff --git a/README.md b/README.md index 6397d14adaad..209d3e21b233 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[PEGASUS-X](https://huggingface.co/docs/transformers/model_doc/pegasus_x)** (from Google) released with the paper [Investigating Efficiently Extending Transformers for Long Input Summarization](https://arxiv.org/abs/2208.04347) by Jason Phang, Yao Zhao, and Peter J. Liu. 1. **[Perceiver IO](https://huggingface.co/docs/transformers/model_doc/perceiver)** (from Deepmind) released with the paper [Perceiver IO: A General Architecture for Structured Inputs & Outputs](https://arxiv.org/abs/2107.14795) by Andrew Jaegle, Sebastian Borgeaud, Jean-Baptiste Alayrac, Carl Doersch, Catalin Ionescu, David Ding, Skanda Koppula, Daniel Zoran, Andrew Brock, Evan Shelhamer, Olivier Hénaff, Matthew M. Botvinick, Andrew Zisserman, Oriol Vinyals, João Carreira. 1. **[PhoBERT](https://huggingface.co/docs/transformers/model_doc/phobert)** (from VinAI Research) released with the paper [PhoBERT: Pre-trained language models for Vietnamese](https://www.aclweb.org/anthology/2020.findings-emnlp.92/) by Dat Quoc Nguyen and Anh Tuan Nguyen. +1. **[Pix2Struct](https://huggingface.co/docs/transformers/main/model_doc/pix2struct)** (from Google) released with the paper [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) by Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. 1. **[PLBart](https://huggingface.co/docs/transformers/model_doc/plbart)** (from UCLA NLP) released with the paper [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) by Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang. 1. **[PoolFormer](https://huggingface.co/docs/transformers/model_doc/poolformer)** (from Sea AI Labs) released with the paper [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) by Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng. 1. **[ProphetNet](https://huggingface.co/docs/transformers/model_doc/prophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. diff --git a/README_es.md b/README_es.md index 899140210cf1..b350d45d40b8 100644 --- a/README_es.md +++ b/README_es.md @@ -387,6 +387,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[PEGASUS-X](https://huggingface.co/docs/transformers/model_doc/pegasus_x)** (from Google) released with the paper [Investigating Efficiently Extending Transformers for Long Input Summarization](https://arxiv.org/abs/2208.04347) by Jason Phang, Yao Zhao, and Peter J. Liu. 1. **[Perceiver IO](https://huggingface.co/docs/transformers/model_doc/perceiver)** (from Deepmind) released with the paper [Perceiver IO: A General Architecture for Structured Inputs & Outputs](https://arxiv.org/abs/2107.14795) by Andrew Jaegle, Sebastian Borgeaud, Jean-Baptiste Alayrac, Carl Doersch, Catalin Ionescu, David Ding, Skanda Koppula, Daniel Zoran, Andrew Brock, Evan Shelhamer, Olivier Hénaff, Matthew M. Botvinick, Andrew Zisserman, Oriol Vinyals, João Carreira. 1. **[PhoBERT](https://huggingface.co/docs/transformers/model_doc/phobert)** (from VinAI Research) released with the paper [PhoBERT: Pre-trained language models for Vietnamese](https://www.aclweb.org/anthology/2020.findings-emnlp.92/) by Dat Quoc Nguyen and Anh Tuan Nguyen. +1. **[Pix2Struct](https://huggingface.co/docs/transformers/main/model_doc/pix2struct)** (from Google) released with the paper [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) by Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. 1. **[PLBart](https://huggingface.co/docs/transformers/model_doc/plbart)** (from UCLA NLP) released with the paper [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) by Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang. 1. **[PoolFormer](https://huggingface.co/docs/transformers/model_doc/poolformer)** (from Sea AI Labs) released with the paper [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) by Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng. 1. **[ProphetNet](https://huggingface.co/docs/transformers/model_doc/prophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. diff --git a/README_hd.md b/README_hd.md index 826306cf67bf..71bf9b61b59c 100644 --- a/README_hd.md +++ b/README_hd.md @@ -359,6 +359,7 @@ conda install -c huggingface transformers 1. **[PEGASUS-X](https://huggingface.co/docs/transformers/model_doc/pegasus_x)** (Google की ओर से) साथ में दिया गया पेपर [लंबे इनपुट सारांश के लिए ट्रांसफ़ॉर्मरों को बेहतर तरीके से एक्सटेंड करना](https://arxiv .org/abs/2208.04347) जेसन फांग, याओ झाओ, पीटर जे लियू द्वारा। 1. **[Perceiver IO](https://huggingface.co/docs/transformers/model_doc/perceiver)** (दीपमाइंड से) साथ में पेपर [पर्सीवर आईओ: संरचित इनपुट और आउटपुट के लिए एक सामान्य वास्तुकला] (https://arxiv.org/abs/2107.14795) एंड्रयू जेगल, सेबेस्टियन बोरग्यूड, जीन-बैप्टिस्ट अलायराक, कार्ल डोर्श, कैटलिन इओनेस्कु, डेविड द्वारा डिंग, स्कंद कोप्पुला, डैनियल ज़ोरान, एंड्रयू ब्रॉक, इवान शेलहैमर, ओलिवियर हेनाफ, मैथ्यू एम। बोट्विनिक, एंड्रयू ज़िसरमैन, ओरिओल विनियल्स, जोआओ कैरेरा द्वारा पोस्ट किया गया। 1. **[PhoBERT](https://huggingface.co/docs/transformers/model_doc/phobert)** (VinAI Research से) कागज के साथ [PhoBERT: वियतनामी के लिए पूर्व-प्रशिक्षित भाषा मॉडल](https://www .aclweb.org/anthology/2020.findings-emnlp.92/) डैट क्वोक गुयेन और अन्ह तुआन गुयेन द्वारा पोस्ट किया गया। +1. **[Pix2Struct](https://huggingface.co/docs/transformers/main/model_doc/pix2struct)** (Google से) Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. द्वाराअनुसंधान पत्र [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) के साथ जारी किया गया 1. **[PLBart](https://huggingface.co/docs/transformers/model_doc/plbart)** (UCLA NLP से) साथ वाला पेपर [प्रोग्राम अंडरस्टैंडिंग एंड जेनरेशन के लिए यूनिफाइड प्री-ट्रेनिंग](https://arxiv .org/abs/2103.06333) वसी उद्दीन अहमद, सैकत चक्रवर्ती, बैशाखी रे, काई-वेई चांग द्वारा। 1. **[PoolFormer](https://huggingface.co/docs/transformers/model_doc/poolformer)** (from Sea AI Labs) released with the paper [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) by Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng. 1. **[ProphetNet](https://huggingface.co/docs/transformers/model_doc/prophetnet)** (माइक्रोसॉफ्ट रिसर्च से) साथ में पेपर [ProphetNet: प्रेडिक्टिंग फ्यूचर एन-ग्राम फॉर सीक्वेंस-टू-सीक्वेंस प्री-ट्रेनिंग ](https://arxiv.org/abs/2001.04063) यू यान, वीज़ेन क्यूई, येयुन गोंग, दयाहेंग लियू, नान डुआन, जिउशेंग चेन, रुओफ़ेई झांग और मिंग झोउ द्वारा पोस्ट किया गया। diff --git a/README_ja.md b/README_ja.md index b45cc68ea6b2..75f45c4462a4 100644 --- a/README_ja.md +++ b/README_ja.md @@ -421,6 +421,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[PEGASUS-X](https://huggingface.co/docs/transformers/model_doc/pegasus_x)** (Google から) Jason Phang, Yao Zhao, and Peter J. Liu から公開された研究論文: [Investigating Efficiently Extending Transformers for Long Input Summarization](https://arxiv.org/abs/2208.04347) 1. **[Perceiver IO](https://huggingface.co/docs/transformers/model_doc/perceiver)** (Deepmind から) Andrew Jaegle, Sebastian Borgeaud, Jean-Baptiste Alayrac, Carl Doersch, Catalin Ionescu, David Ding, Skanda Koppula, Daniel Zoran, Andrew Brock, Evan Shelhamer, Olivier Hénaff, Matthew M. Botvinick, Andrew Zisserman, Oriol Vinyals, João Carreira から公開された研究論文: [Perceiver IO: A General Architecture for Structured Inputs & Outputs](https://arxiv.org/abs/2107.14795) 1. **[PhoBERT](https://huggingface.co/docs/transformers/model_doc/phobert)** (VinAI Research から) Dat Quoc Nguyen and Anh Tuan Nguyen から公開された研究論文: [PhoBERT: Pre-trained language models for Vietnamese](https://www.aclweb.org/anthology/2020.findings-emnlp.92/) +1. **[Pix2Struct](https://huggingface.co/docs/transformers/main/model_doc/pix2struct)** (Google から) Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. から公開された研究論文 [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) 1. **[PLBart](https://huggingface.co/docs/transformers/model_doc/plbart)** (UCLA NLP から) Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang から公開された研究論文: [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) 1. **[PoolFormer](https://huggingface.co/docs/transformers/model_doc/poolformer)** (Sea AI Labs から) Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng から公開された研究論文: [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) 1. **[ProphetNet](https://huggingface.co/docs/transformers/model_doc/prophetnet)** (Microsoft Research から) Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou から公開された研究論文: [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) diff --git a/README_ko.md b/README_ko.md index a5c0b8cf1eee..703c898707fa 100644 --- a/README_ko.md +++ b/README_ko.md @@ -336,6 +336,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[PEGASUS-X](https://huggingface.co/docs/transformers/model_doc/pegasus_x)** (Google 에서) Jason Phang, Yao Zhao, Peter J. Liu 의 [Investigating Efficiently Extending Transformers for Long Input Summarization](https://arxiv.org/abs/2208.04347) 논문과 함께 발표했습니다. 1. **[Perceiver IO](https://huggingface.co/docs/transformers/model_doc/perceiver)** (Deepmind 에서) Andrew Jaegle, Sebastian Borgeaud, Jean-Baptiste Alayrac, Carl Doersch, Catalin Ionescu, David Ding, Skanda Koppula, Daniel Zoran, Andrew Brock, Evan Shelhamer, Olivier Hénaff, Matthew M. Botvinick, Andrew Zisserman, Oriol Vinyals, João Carreira 의 [Perceiver IO: A General Architecture for Structured Inputs & Outputs](https://arxiv.org/abs/2107.14795) 논문과 함께 발표했습니다. 1. **[PhoBERT](https://huggingface.co/docs/transformers/model_doc/phobert)** (VinAI Research 에서) Dat Quoc Nguyen and Anh Tuan Nguyen 의 [PhoBERT: Pre-trained language models for Vietnamese](https://www.aclweb.org/anthology/2020.findings-emnlp.92/) 논문과 함께 발표했습니다. +1. **[Pix2Struct](https://huggingface.co/docs/transformers/main/model_doc/pix2struct)** (Google 에서 제공)은 Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova.의 [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347)논문과 함께 발표했습니다. 1. **[PLBart](https://huggingface.co/docs/transformers/model_doc/plbart)** (UCLA NLP 에서) Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang 의 [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) 논문과 함께 발표했습니다. 1. **[PoolFormer](https://huggingface.co/docs/transformers/model_doc/poolformer)** (Sea AI Labs 에서) Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng 의 [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) 논문과 함께 발표했습니다. 1. **[ProphetNet](https://huggingface.co/docs/transformers/model_doc/prophetnet)** (Microsoft Research 에서) Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou 의 [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) 논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index 9ae3bc24494f..8fa869b498c7 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -360,6 +360,7 @@ conda install -c huggingface transformers 1. **[PEGASUS-X](https://huggingface.co/docs/transformers/model_doc/pegasus_x)** (来自 Google) 伴随论文 [Investigating Efficiently Extending Transformers for Long Input Summarization](https://arxiv.org/abs/2208.04347) 由 Jason Phang, Yao Zhao, Peter J. Liu 发布。 1. **[Perceiver IO](https://huggingface.co/docs/transformers/model_doc/perceiver)** (来自 Deepmind) 伴随论文 [Perceiver IO: A General Architecture for Structured Inputs & Outputs](https://arxiv.org/abs/2107.14795) 由 Andrew Jaegle, Sebastian Borgeaud, Jean-Baptiste Alayrac, Carl Doersch, Catalin Ionescu, David Ding, Skanda Koppula, Daniel Zoran, Andrew Brock, Evan Shelhamer, Olivier Hénaff, Matthew M. Botvinick, Andrew Zisserman, Oriol Vinyals, João Carreira 发布。 1. **[PhoBERT](https://huggingface.co/docs/transformers/model_doc/phobert)** (来自 VinAI Research) 伴随论文 [PhoBERT: Pre-trained language models for Vietnamese](https://www.aclweb.org/anthology/2020.findings-emnlp.92/) 由 Dat Quoc Nguyen and Anh Tuan Nguyen 发布。 +1. **[Pix2Struct](https://huggingface.co/docs/transformers/main/model_doc/pix2struct)** (来自 Google) 伴随论文 [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) 由 Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova 发布。 1. **[PLBart](https://huggingface.co/docs/transformers/model_doc/plbart)** (来自 UCLA NLP) 伴随论文 [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) 由 Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang 发布。 1. **[PoolFormer](https://huggingface.co/docs/transformers/model_doc/poolformer)** (来自 Sea AI Labs) 伴随论文 [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) 由 Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng 发布。 1. **[ProphetNet](https://huggingface.co/docs/transformers/model_doc/prophetnet)** (来自 Microsoft Research) 伴随论文 [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) 由 Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index 53847bf6739a..eae33654a950 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -372,6 +372,7 @@ conda install -c huggingface transformers 1. **[PEGASUS-X](https://huggingface.co/docs/transformers/model_doc/pegasus_x)** (from Google) released with the paper [Investigating Efficiently Extending Transformers for Long Input Summarization](https://arxiv.org/abs/2208.04347) by Jason Phang, Yao Zhao, Peter J. Liu. 1. **[Perceiver IO](https://huggingface.co/docs/transformers/model_doc/perceiver)** (from Deepmind) released with the paper [Perceiver IO: A General Architecture for Structured Inputs & Outputs](https://arxiv.org/abs/2107.14795) by Andrew Jaegle, Sebastian Borgeaud, Jean-Baptiste Alayrac, Carl Doersch, Catalin Ionescu, David Ding, Skanda Koppula, Daniel Zoran, Andrew Brock, Evan Shelhamer, Olivier Hénaff, Matthew M. Botvinick, Andrew Zisserman, Oriol Vinyals, João Carreira. 1. **[PhoBERT](https://huggingface.co/docs/transformers/model_doc/phobert)** (from VinAI Research) released with the paper [PhoBERT: Pre-trained language models for Vietnamese](https://www.aclweb.org/anthology/2020.findings-emnlp.92/) by Dat Quoc Nguyen and Anh Tuan Nguyen. +1. **[Pix2Struct](https://huggingface.co/docs/transformers/main/model_doc/pix2struct)** (from Google) released with the paper [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) by Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. 1. **[PLBart](https://huggingface.co/docs/transformers/model_doc/plbart)** (from UCLA NLP) released with the paper [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) by Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang. 1. **[PoolFormer](https://huggingface.co/docs/transformers/model_doc/poolformer)** (from Sea AI Labs) released with the paper [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) by Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng. 1. **[ProphetNet](https://huggingface.co/docs/transformers/model_doc/prophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 7586985b1112..c2e6933d44d8 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -596,6 +596,8 @@ title: OWL-ViT - local: model_doc/perceiver title: Perceiver + - local: model_doc/pix2struct + title: Pix2Struct - local: model_doc/speech-encoder-decoder title: Speech Encoder Decoder Models - local: model_doc/tapas diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index ea1ab27e7970..eee27503451c 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -173,6 +173,7 @@ The documentation is organized into five sections: 1. **[PEGASUS-X](model_doc/pegasus_x)** (from Google) released with the paper [Investigating Efficiently Extending Transformers for Long Input Summarization](https://arxiv.org/abs/2208.04347) by Jason Phang, Yao Zhao, and Peter J. Liu. 1. **[Perceiver IO](model_doc/perceiver)** (from Deepmind) released with the paper [Perceiver IO: A General Architecture for Structured Inputs & Outputs](https://arxiv.org/abs/2107.14795) by Andrew Jaegle, Sebastian Borgeaud, Jean-Baptiste Alayrac, Carl Doersch, Catalin Ionescu, David Ding, Skanda Koppula, Daniel Zoran, Andrew Brock, Evan Shelhamer, Olivier Hénaff, Matthew M. Botvinick, Andrew Zisserman, Oriol Vinyals, João Carreira. 1. **[PhoBERT](model_doc/phobert)** (from VinAI Research) released with the paper [PhoBERT: Pre-trained language models for Vietnamese](https://www.aclweb.org/anthology/2020.findings-emnlp.92/) by Dat Quoc Nguyen and Anh Tuan Nguyen. +1. **[Pix2Struct](model_doc/pix2struct)** (from Google) released with the paper [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) by Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. 1. **[PLBart](model_doc/plbart)** (from UCLA NLP) released with the paper [Unified Pre-training for Program Understanding and Generation](https://arxiv.org/abs/2103.06333) by Wasi Uddin Ahmad, Saikat Chakraborty, Baishakhi Ray, Kai-Wei Chang. 1. **[PoolFormer](model_doc/poolformer)** (from Sea AI Labs) released with the paper [MetaFormer is Actually What You Need for Vision](https://arxiv.org/abs/2111.11418) by Yu, Weihao and Luo, Mi and Zhou, Pan and Si, Chenyang and Zhou, Yichen and Wang, Xinchao and Feng, Jiashi and Yan, Shuicheng. 1. **[ProphetNet](model_doc/prophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. @@ -364,6 +365,7 @@ Flax), PyTorch, and/or TensorFlow. | Pegasus | ✅ | ✅ | ✅ | ✅ | ✅ | | PEGASUS-X | ❌ | ❌ | ✅ | ❌ | ❌ | | Perceiver | ✅ | ❌ | ✅ | ❌ | ❌ | +| Pix2Struct | ❌ | ❌ | ✅ | ❌ | ❌ | | PLBart | ✅ | ❌ | ✅ | ❌ | ❌ | | PoolFormer | ❌ | ❌ | ✅ | ❌ | ❌ | | ProphetNet | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/model_doc/pix2struct.mdx b/docs/source/en/model_doc/pix2struct.mdx new file mode 100644 index 000000000000..340b06c69b07 --- /dev/null +++ b/docs/source/en/model_doc/pix2struct.mdx @@ -0,0 +1,73 @@ + + +# Pix2Struct + +## Overview + +The Pix2Struct model was proposed in [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language Understanding](https://arxiv.org/abs/2210.03347) by Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. + +The abstract from the paper is the following: + +> Visually-situated language is ubiquitous -- sources range from textbooks with diagrams to web pages with images and tables, to mobile apps with buttons and forms. Perhaps due to this diversity, previous work has typically relied on domain-specific recipes with limited sharing of the underlying data, model architectures, and objectives. We present Pix2Struct, a pretrained image-to-text model for purely visual language understanding, which can be finetuned on tasks containing visually-situated language. Pix2Struct is pretrained by learning to parse masked screenshots of web pages into simplified HTML. The web, with its richness of visual elements cleanly reflected in the HTML structure, provides a large source of pretraining data well suited to the diversity of downstream tasks. Intuitively, this objective subsumes common pretraining signals such as OCR, language modeling, image captioning. In addition to the novel pretraining strategy, we introduce a variable-resolution input representation and a more flexible integration of language and vision inputs, where language prompts such as questions are rendered directly on top of the input image. For the first time, we show that a single pretrained model can achieve state-of-the-art results in six out of nine tasks across four domains: documents, illustrations, user interfaces, and natural images. + +Tips: + +Pix2Struct has been fine tuned on a variety of tasks and datasets, ranging from image captioning, visual question answering (VQA) over different inputs (books, charts, science diagrams), captioning UI components etc. The full list can be found in Table 1 of the paper. +We therefore advise you to use these models for the tasks they have been fine tuned on. For instance, if you want to use Pix2Struct for UI captioning, you should use the model fine tuned on the UI dataset. If you want to use Pix2Struct for image captioning, you should use the model fine tuned on the natural images captioning dataset and so on. + +This model was contributed by [ybelkada](https://huggingface.co/ybelkada). +The original code can be found [here](https://github.com/google-research/pix2struct). + +## Resources: + +- [Paper](https://arxiv.org/abs/2210.03347) +- [Fine-tuning Notebook](https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb) +- [All models](https://huggingface.co/models?search=pix2struct) + + +## Pix2StructConfig + +[[autodoc]] Pix2StructConfig + - from_text_vision_configs + +## Pix2StructTextConfig + +[[autodoc]] Pix2StructTextConfig + +## Pix2StructVisionConfig + +[[autodoc]] Pix2StructVisionConfig + +## Pix2StructProcessor + +[[autodoc]] Pix2StructProcessor + +## Pix2StructImageProcessor + +[[autodoc]] Pix2StructImageProcessor + - preprocess + +## Pix2StructTextModel + +[[autodoc]] Pix2StructTextModel + - forward + +## Pix2StructVisionModel + +[[autodoc]] Pix2StructVisionModel + - forward + +## Pix2StructForConditionalGeneration + +[[autodoc]] Pix2StructForConditionalGeneration + - forward diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 7b70b4ac1f3b..894ca8e40d68 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -403,6 +403,13 @@ "models.pegasus_x": ["PEGASUS_X_PRETRAINED_CONFIG_ARCHIVE_MAP", "PegasusXConfig"], "models.perceiver": ["PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP", "PerceiverConfig", "PerceiverTokenizer"], "models.phobert": ["PhobertTokenizer"], + "models.pix2struct": [ + "PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP", + "Pix2StructConfig", + "Pix2StructProcessor", + "Pix2StructTextConfig", + "Pix2StructVisionConfig", + ], "models.plbart": ["PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP", "PLBartConfig"], "models.poolformer": ["POOLFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "PoolFormerConfig"], "models.prophetnet": ["PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "ProphetNetConfig", "ProphetNetTokenizer"], @@ -861,6 +868,7 @@ _import_structure["models.oneformer"].extend(["OneFormerImageProcessor"]) _import_structure["models.owlvit"].extend(["OwlViTFeatureExtractor", "OwlViTImageProcessor"]) _import_structure["models.perceiver"].extend(["PerceiverFeatureExtractor", "PerceiverImageProcessor"]) + _import_structure["models.pix2struct"].extend(["Pix2StructImageProcessor"]) _import_structure["models.poolformer"].extend(["PoolFormerFeatureExtractor", "PoolFormerImageProcessor"]) _import_structure["models.segformer"].extend(["SegformerFeatureExtractor", "SegformerImageProcessor"]) _import_structure["models.swin2sr"].append("Swin2SRImageProcessor") @@ -2101,6 +2109,15 @@ "PerceiverPreTrainedModel", ] ) + _import_structure["models.pix2struct"].extend( + [ + "PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST", + "Pix2StructForConditionalGeneration", + "Pix2StructPreTrainedModel", + "Pix2StructTextModel", + "Pix2StructVisionModel", + ] + ) _import_structure["models.plbart"].extend( [ "PLBART_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -4015,6 +4032,13 @@ from .models.pegasus_x import PEGASUS_X_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusXConfig from .models.perceiver import PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP, PerceiverConfig, PerceiverTokenizer from .models.phobert import PhobertTokenizer + from .models.pix2struct import ( + PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP, + Pix2StructConfig, + Pix2StructProcessor, + Pix2StructTextConfig, + Pix2StructVisionConfig, + ) from .models.plbart import PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP, PLBartConfig from .models.poolformer import POOLFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, PoolFormerConfig from .models.prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig, ProphetNetTokenizer @@ -4420,6 +4444,7 @@ from .models.oneformer import OneFormerImageProcessor from .models.owlvit import OwlViTFeatureExtractor, OwlViTImageProcessor from .models.perceiver import PerceiverFeatureExtractor, PerceiverImageProcessor + from .models.pix2struct import Pix2StructImageProcessor from .models.poolformer import PoolFormerFeatureExtractor, PoolFormerImageProcessor from .models.segformer import SegformerFeatureExtractor, SegformerImageProcessor from .models.swin2sr import Swin2SRImageProcessor @@ -5436,6 +5461,13 @@ PerceiverModel, PerceiverPreTrainedModel, ) + from .models.pix2struct import ( + PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST, + Pix2StructForConditionalGeneration, + Pix2StructPreTrainedModel, + Pix2StructTextModel, + Pix2StructVisionModel, + ) from .models.plbart import ( PLBART_PRETRAINED_MODEL_ARCHIVE_LIST, PLBartForCausalLM, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index d366eefdcbb0..d32aa2f6e884 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -140,6 +140,7 @@ pegasus_x, perceiver, phobert, + pix2struct, plbart, poolformer, prophetnet, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 9a8ef79c7c11..9a8500d118ff 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -141,6 +141,7 @@ ("pegasus", "PegasusConfig"), ("pegasus_x", "PegasusXConfig"), ("perceiver", "PerceiverConfig"), + ("pix2struct", "Pix2StructConfig"), ("plbart", "PLBartConfig"), ("poolformer", "PoolFormerConfig"), ("prophetnet", "ProphetNetConfig"), @@ -315,6 +316,7 @@ ("pegasus", "PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("pegasus_x", "PEGASUS_X_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("perceiver", "PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("pix2struct", "PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("plbart", "PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("poolformer", "POOLFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("prophetnet", "PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -505,6 +507,7 @@ ("pegasus_x", "PEGASUS-X"), ("perceiver", "Perceiver"), ("phobert", "PhoBERT"), + ("pix2struct", "Pix2Struct"), ("plbart", "PLBart"), ("poolformer", "PoolFormer"), ("prophetnet", "ProphetNet"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 95ecd57172b4..c092dbf16f46 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -80,6 +80,7 @@ ("oneformer", "OneFormerImageProcessor"), ("owlvit", "OwlViTImageProcessor"), ("perceiver", "PerceiverImageProcessor"), + ("pix2struct", "Pix2StructImageProcessor"), ("poolformer", "PoolFormerImageProcessor"), ("regnet", "ConvNextImageProcessor"), ("resnet", "ConvNextImageProcessor"), diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index fb020034008b..9e6edc0ae16f 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -60,6 +60,7 @@ ("mgp-str", "MgpstrProcessor"), ("oneformer", "OneFormerProcessor"), ("owlvit", "OwlViTProcessor"), + ("pix2struct", "Pix2StructProcessor"), ("sew", "Wav2Vec2Processor"), ("sew-d", "Wav2Vec2Processor"), ("speech_to_text", "Speech2TextProcessor"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index a029bfb52cd6..5afedadca446 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -248,6 +248,7 @@ ), ), ("phobert", ("PhobertTokenizer", None)), + ("pix2struct", ("T5Tokenizer", "T5TokenizerFast" if is_tokenizers_available() else None)), ("plbart", ("PLBartTokenizer" if is_sentencepiece_available() else None, None)), ("prophetnet", ("ProphetNetTokenizer", None)), ("qdqbert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), diff --git a/src/transformers/models/pix2struct/__init__.py b/src/transformers/models/pix2struct/__init__.py new file mode 100644 index 000000000000..8b395b31d8be --- /dev/null +++ b/src/transformers/models/pix2struct/__init__.py @@ -0,0 +1,86 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available + + +_import_structure = { + "configuration_pix2struct": [ + "PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP", + "Pix2StructConfig", + "Pix2StructTextConfig", + "Pix2StructVisionConfig", + ], + "processing_pix2struct": ["Pix2StructProcessor"], +} + +try: + if not is_vision_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["image_processing_pix2struct"] = ["Pix2StructImageProcessor"] + + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_pix2struct"] = [ + "PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST", + "Pix2StructPreTrainedModel", + "Pix2StructForConditionalGeneration", + "Pix2StructVisionModel", + "Pix2StructTextModel", + ] + +if TYPE_CHECKING: + from .configuration_pix2struct import ( + PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP, + Pix2StructConfig, + Pix2StructTextConfig, + Pix2StructVisionConfig, + ) + from .processing_pix2struct import Pix2StructProcessor + + try: + if not is_vision_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .image_processing_pix2struct import Pix2StructImageProcessor + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_pix2struct import ( + PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST, + Pix2StructForConditionalGeneration, + Pix2StructPreTrainedModel, + Pix2StructTextModel, + Pix2StructVisionModel, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/pix2struct/configuration_pix2struct.py b/src/transformers/models/pix2struct/configuration_pix2struct.py new file mode 100644 index 000000000000..c8a866966cdf --- /dev/null +++ b/src/transformers/models/pix2struct/configuration_pix2struct.py @@ -0,0 +1,412 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Pix2Struct model configuration""" + +import copy +import os +from typing import Union + +from ...configuration_utils import PretrainedConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "google/pix2struct-textcaps-base": ( + "https://huggingface.co/google/pix2struct-textcaps-base/resolve/main/config.json" + ), +} + + +class Pix2StructTextConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Pix2StructTextModel`]. It is used to instantiate + a Pix2Struct text model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the `Pix2StructText` used by the + [base architectures](https://huggingface.co/google/pix2struct-textcaps-base). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 50244): + Vocabulary size of the `Pix2Struct` text model. Defines the number of different tokens that can be + represented by the `inputs_ids` passed when calling [`Pix2StructModel`]. + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + d_kv (`int`, *optional*, defaults to 64): + Dimensionality of the key, query, value projections in each attention head. + d_ff (`int`, *optional*, defaults to 2048): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + num_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + relative_attention_num_buckets (`int`, *optional*, defaults to 32): + The number of buckets to use for each attention layer. + relative_attention_max_distance (`int`, *optional*, defaults to 128): + The maximum distance of the longer sequences for the bucket separation. + dropout_rate (`float`, *optional*, defaults to 0.1): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + layer_norm_epsilon (`float`, *optional*, defaults to 1e-6): + The epsilon used by the layer normalization layers. + initializer_factor (`float`, *optional*, defaults to 1.0): + A factor for initializing all weight matrices (should be kept to 1, used internally for initialization + testing). + dense_act_fn (`Union[Callable, str]`, *optional*, defaults to `"gelu_new"`): + The non-linear activation function (function or string). + decoder_start_token_id (`int`, *optional*, defaults to 0): + The id of the `decoder_start_token_id` token. + use_cache (`bool`, *optional*, defaults to `False`): + Whether or not the model should return the last key/values attentions (not used by all models). + pad_token_id (`int`, *optional*, defaults to 0): + The id of the `padding` token. + eos_token_id (`int`, *optional*, defaults to 1): + The id of the `end-of-sequence` token. + + Example: + + ```python + >>> from transformers import Pix2StructTextConfig, Pix2StructTextModel + + >>> # Initializing a Pix2StructTextConfig with Salesforce/pix2struct-vqa-base style configuration + >>> configuration = Pix2StructTextConfig() + + >>> # Initializing a Pix2StructTextModel (with random weights) from the Salesforce/pix2struct-vqa-base style configuration + >>> model = Pix2StructTextModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "pix2struct_text_model" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = { + "hidden_size": "hidden_size", + "num_attention_heads": "num_heads", + "num_hidden_layers": "num_layers", + } + + def __init__( + self, + vocab_size=50244, + hidden_size=768, + d_kv=64, + d_ff=2048, + num_layers=12, + num_heads=12, + relative_attention_num_buckets=32, + relative_attention_max_distance=128, + dropout_rate=0.1, + layer_norm_epsilon=1e-6, + initializer_factor=1.0, + dense_act_fn="gelu_new", + decoder_start_token_id=0, + use_cache=False, + pad_token_id=0, + eos_token_id=1, + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.d_kv = d_kv + self.d_ff = d_ff + self.num_layers = num_layers + self.num_heads = num_heads + self.relative_attention_num_buckets = relative_attention_num_buckets + self.relative_attention_max_distance = relative_attention_max_distance + self.dropout_rate = dropout_rate + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_factor = initializer_factor + self.use_cache = use_cache + + self.eos_token_id = eos_token_id + self.decoder_start_token_id = decoder_start_token_id + + # for backwards compatibility + self.dense_act_fn = dense_act_fn + + super().__init__( + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + decoder_start_token_id=decoder_start_token_id, + **kwargs, + ) + + @classmethod + def from_pretrained( + cls, pretrainehidden_size_name_or_path: Union[str, os.PathLike], **kwargs + ) -> "PretrainedConfig": + config_dict, kwargs = cls.get_config_dict(pretrainehidden_size_name_or_path, **kwargs) + + # get the text config dict if we are loading from Pix2StructConfig + if config_dict.get("model_type") == "pix2struct": + config_dict = config_dict["text_config"] + + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class Pix2StructVisionConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Pix2StructVisionModel`]. It is used to + instantiate a PIX2STRUCT vision model according to the specified arguments, defining the model architecture. + Instantiating a configuration defaults will yield a similar configuration to that of the Pix2Struct-base + [Salesforce/pix2struct-vqa-base](https://huggingface.co/Salesforce/pix2struct-vqa-base) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + patch_embed_hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the input patch_embedding layer in the Transformer encoder. + d_ff (`int`, *optional*, defaults to 2048): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + d_kv (`int`, *optional*, defaults to 64): + Dimensionality of the key, query, value projections per attention head. + projection_dim (`int`, *optional*, defaults to 768): + Dimensionality of the projection layer in the Transformer encoder. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + num_channels (`int`, *optional*, defaults to 3): + Number of channels of the input images. + patch_size (`int`, *optional*, defaults to 16): + The size (resolution) of each patch. + dense_act_fn (`str` or `function`, *optional*, defaults to `"gelu_new"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. + layer_norm_eps (`float`, *optional*, defaults to 1e-6): + The epsilon used by the layer normalization layers. + dropout_rate (`float`, *optional*, defaults to 0.0): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 1e-10): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float``, *optional*, defaults to 1): + A factor for initializing all weight matrices (should be kept to 1, used internally for initialization + testing). + seq_len (`int`, *optional*, defaults to 4096): + Maximum sequence length (here number of patches) supported by the model. + layer_norm_bias (`bool`, *optional*, defaults to `False`): + Whether or not to add a bias to the layer normalization layers. + relative_attention_num_buckets (`int`, *optional*, defaults to 32): + The number of buckets to use for each attention layer. + relative_attention_max_distance (`int`, *optional*, defaults to 128): + The maximum distance (in tokens) to use for each attention layer. + + Example: + + ```python + >>> from transformers import Pix2StructVisionConfig, Pix2StructVisionModel + + >>> # Initializing a Pix2StructVisionConfig with Salesforce/pix2struct-vqa-base style configuration + >>> configuration = Pix2StructVisionConfig() + + >>> # Initializing a Pix2StructVisionModel (with random weights) from the Salesforce/pix2struct-vqa-base style configuration + >>> model = Pix2StructVisionModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "pix2struct_vision_model" + + def __init__( + self, + hidden_size=768, + patch_embed_hidden_size=768, + d_ff=2048, + d_kv=64, + projection_dim=768, + num_hidden_layers=12, + num_attention_heads=12, + num_channels=3, + patch_size=16, + dense_act_fn="gelu_new", + layer_norm_eps=1e-6, + dropout_rate=0.0, + attention_dropout=0.0, + initializer_range=1e-10, + initializer_factor=1.0, + seq_len=4096, + layer_norm_bias=False, + relative_attention_num_buckets=32, + relative_attention_max_distance=128, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.patch_embed_hidden_size = patch_embed_hidden_size + self.d_ff = d_ff + self.projection_dim = projection_dim + self.dropout_rate = dropout_rate + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + self.layer_norm_eps = layer_norm_eps + self.dense_act_fn = dense_act_fn + self.seq_len = seq_len + self.layer_norm_bias = layer_norm_bias + self.relative_attention_num_buckets = relative_attention_num_buckets + self.relative_attention_max_distance = relative_attention_max_distance + self.d_kv = d_kv + + @classmethod + def from_pretrained( + cls, pretrainehidden_size_name_or_path: Union[str, os.PathLike], **kwargs + ) -> "PretrainedConfig": + config_dict, kwargs = cls.get_config_dict(pretrainehidden_size_name_or_path, **kwargs) + + # get the vision config dict if we are loading from Pix2StructConfig + if config_dict.get("model_type") == "pix2struct": + config_dict = config_dict["vision_config"] + + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class Pix2StructConfig(PretrainedConfig): + r""" + [`Pix2StructConfig`] is the configuration class to store the configuration of a [`Pix2StructModel`]. It is used to + instantiate a PIX2STRUCT model according to the specified arguments, defining the text model and vision model + configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the + PIX2STRUCT-base [Salesforce/pix2struct-vqa-base](https://huggingface.co/Salesforce/pix2struct-vqa-base) + architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + text_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`Pix2StructTextConfig`]. + vision_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`Pix2StructVisionConfig`]. + initializer_factor (`float`, *optional*, defaults to 1.0): + Factor to multiply the initialization range with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + is_vqa (`bool`, *optional*, defaults to `False`): + Whether the model has been fine-tuned for VQA or not. + kwargs (*optional*): + Dictionary of keyword arguments. + + Example: + + ```python + >>> from transformers import Pix2StructConfig, Pix2StructModel + + >>> # Initializing a Pix2StructConfig with Salesforce/pix2struct-vqa-base style configuration + >>> configuration = Pix2StructConfig() + + >>> # Initializing a Pix2StructPModel (with random weights) from the Salesforce/pix2struct-vqa-base style configuration + >>> model = Pix2StructModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + + >>> # We can also initialize a Pix2StructConfig from a Pix2StructTextConfig and a Pix2StructVisionConfig + + >>> # Initializing a PIX2STRUCTText and PIX2STRUCTVision configuration + >>> config_text = Pix2StructTextConfig() + >>> config_vision = Pix2StructVisionConfig() + + >>> config = Pix2StructConfig.from_text_vision_configs(config_text, config_vision) + ```""" + + model_type = "pix2struct" + is_composition = True + + def __init__( + self, + text_config=None, + vision_config=None, + initializer_factor=1.0, + initializer_range=0.02, + is_vqa=False, + **kwargs, + ): + super().__init__(**kwargs) + + if text_config is None: + text_config = {} + logger.info("text_config is None. Initializing the Pix2StructTextConfig with default values.") + + if vision_config is None: + vision_config = {} + logger.info("vision_config is None. Initializing the Pix2StructVisionConfig with default values.") + + self.text_config = Pix2StructTextConfig(**text_config) + self.vision_config = Pix2StructVisionConfig(**vision_config) + + self.text_config.encoder_hidden_size = self.vision_config.hidden_size + self.decoder_start_token_id = self.text_config.decoder_start_token_id + self.pad_token_id = self.text_config.pad_token_id + + self.initializer_factor = initializer_factor + self.initializer_range = initializer_range + + self.text_config.initializer_range = self.initializer_range + self.vision_config.initializer_range = self.initializer_range + + self.is_vqa = is_vqa + + @classmethod + def from_text_vision_configs( + cls, text_config: Pix2StructTextConfig, vision_config: Pix2StructVisionConfig, **kwargs + ): + r""" + Instantiate a [`Pix2StructConfig`] (or a derived class) from pix2struct text model configuration and pix2struct + vision model configuration. + + Returns: + [`Pix2StructConfig`]: An instance of a configuration object + """ + + return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs) + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output["text_config"] = self.text_config.to_dict() + output["vision_config"] = self.vision_config.to_dict() + output["model_type"] = self.__class__.model_type + return output diff --git a/src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py b/src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py new file mode 100644 index 000000000000..457c2236694a --- /dev/null +++ b/src/transformers/models/pix2struct/convert_pix2struct_original_pytorch_to_hf.py @@ -0,0 +1,155 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import re + +import torch +from flax.traverse_util import flatten_dict +from t5x import checkpoints + +from transformers import ( + AutoTokenizer, + Pix2StructConfig, + Pix2StructForConditionalGeneration, + Pix2StructImageProcessor, + Pix2StructProcessor, + Pix2StructTextConfig, + Pix2StructVisionConfig, +) + + +def get_flax_param(t5x_checkpoint_path): + flax_params = checkpoints.load_t5x_checkpoint(t5x_checkpoint_path) + flax_params = flatten_dict(flax_params) + return flax_params + + +def rename_and_convert_flax_params(flax_dict): + converted_dict = {} + + CONVERSION_MAPPING = { + "token_embedder": "embeddings", + "encoder_norm": "layernorm", + "kernel": "weight", + ".out": ".output", + "scale": "weight", + "embedders_0.pos_embedding": "row_embedder.weight", + "embedders_1.pos_embedding": "column_embedder.weight", + } + + DECODER_CONVERSION_MAPPING = { + "query": "attention.query", + "key": "attention.key", + "value": "attention.value", + "output.dense": "output", + "encoder_decoder_attention.o": "encoder_decoder_attention.attention.o", + "pre_self_attention_layer_norm": "self_attention.layer_norm", + "pre_cross_attention_layer_norm": "encoder_decoder_attention.layer_norm", + "mlp.": "mlp.DenseReluDense.", + "pre_mlp_layer_norm": "mlp.layer_norm", + "self_attention.o": "self_attention.attention.o", + "decoder.embeddings.embedding": "decoder.embed_tokens.weight", + "decoder.relpos_bias.rel_embedding": "decoder.layer.0.self_attention.attention.relative_attention_bias.weight", + "decoder.decoder_norm.weight": "decoder.final_layer_norm.weight", + "decoder.logits_dense.weight": "decoder.lm_head.weight", + } + + for key in flax_dict.keys(): + if "target" in key: + # remove the first prefix from the key + new_key = ".".join(key[1:]) + + # rename the key + for old, new in CONVERSION_MAPPING.items(): + new_key = new_key.replace(old, new) + + if "decoder" in new_key: + for old, new in DECODER_CONVERSION_MAPPING.items(): + new_key = new_key.replace(old, new) + + if "layers" in new_key and "decoder" not in new_key: + # use regex to replace the layer number + new_key = re.sub(r"layers_(\d+)", r"layer.\1", new_key) + new_key = new_key.replace("encoder", "encoder.encoder") + + elif "layers" in new_key and "decoder" in new_key: + # use regex to replace the layer number + new_key = re.sub(r"layers_(\d+)", r"layer.\1", new_key) + + converted_dict[new_key] = flax_dict[key] + + converted_torch_dict = {} + # convert converted_dict into torch format + for key in converted_dict.keys(): + if ("embed_tokens" not in key) and ("embedder" not in key): + converted_torch_dict[key] = torch.from_numpy(converted_dict[key].T) + else: + converted_torch_dict[key] = torch.from_numpy(converted_dict[key]) + + return converted_torch_dict + + +def convert_pix2struct_original_pytorch_checkpoint_to_hf( + t5x_checkpoint_path, pytorch_dump_folder_path, use_large=False, is_vqa=False +): + flax_params = get_flax_param(t5x_checkpoint_path) + + if not use_large: + encoder_config = Pix2StructVisionConfig() + decoder_config = Pix2StructTextConfig() + else: + encoder_config = Pix2StructVisionConfig( + hidden_size=1536, d_ff=3968, num_attention_heads=24, num_hidden_layers=18 + ) + decoder_config = Pix2StructTextConfig(hidden_size=1536, d_ff=3968, num_heads=24, num_layers=18) + config = Pix2StructConfig( + vision_config=encoder_config.to_dict(), text_config=decoder_config.to_dict(), is_vqa=is_vqa + ) + + model = Pix2StructForConditionalGeneration(config) + + torch_params = rename_and_convert_flax_params(flax_params) + model.load_state_dict(torch_params) + + tok = AutoTokenizer.from_pretrained("ybelkada/test-pix2struct-tokenizer") + image_processor = Pix2StructImageProcessor() + processor = Pix2StructProcessor(image_processor=image_processor, tokenizer=tok) + + if use_large: + processor.image_processor.max_patches = 4096 + + processor.image_processor.is_vqa = True + + # mkdir if needed + os.makedirs(pytorch_dump_folder_path, exist_ok=True) + + model.save_pretrained(pytorch_dump_folder_path) + processor.save_pretrained(pytorch_dump_folder_path) + + print("Model saved in {}".format(pytorch_dump_folder_path)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--t5x_checkpoint_path", default=None, type=str, help="Path to the original T5x checkpoint.") + parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") + parser.add_argument("--use_large", action="store_true", help="Use large model.") + parser.add_argument("--is_vqa", action="store_true", help="Use large model.") + args = parser.parse_args() + + convert_pix2struct_original_pytorch_checkpoint_to_hf( + args.t5x_checkpoint_path, args.pytorch_dump_folder_path, args.use_large + ) diff --git a/src/transformers/models/pix2struct/image_processing_pix2struct.py b/src/transformers/models/pix2struct/image_processing_pix2struct.py new file mode 100644 index 000000000000..b85d1238417e --- /dev/null +++ b/src/transformers/models/pix2struct/image_processing_pix2struct.py @@ -0,0 +1,418 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Image processor class for Pix2Struct.""" +import io +import math +from typing import Dict, Optional, Union + +import numpy as np +from huggingface_hub import hf_hub_download + +from ...image_processing_utils import BaseImageProcessor, BatchFeature +from ...image_transforms import convert_to_rgb, normalize, to_channel_dimension_format, to_pil_image +from ...image_utils import ( + ChannelDimension, + ImageInput, + get_image_size, + infer_channel_dimension_format, + make_list_of_images, + to_numpy_array, + valid_images, +) +from ...utils import TensorType, is_torch_available, is_vision_available, logging +from ...utils.import_utils import requires_backends + + +if is_vision_available(): + import textwrap + + from PIL import Image, ImageDraw, ImageFont + +if is_torch_available(): + import torch + + +logger = logging.get_logger(__name__) +DEFAULT_FONT_PATH = "ybelkada/fonts" + + +# adapted from: https://discuss.pytorch.org/t/tf-image-extract-patches-in-pytorch/171409/2 +def torch_extract_patches(image_tensor, patch_height, patch_width): + """ + Utiliy function to extract patches from a given image tensor. Returns a tensor of shape (1, `patch_height`, + `patch_width`, `num_channels`x `patch_height` x `patch_width`) + + Args: + image_tensor (torch.Tensor): + The image tensor to extract patches from. + patch_height (int): + The height of the patches to extract. + patch_width (int): + The width of the patches to extract. + """ + requires_backends(torch_extract_patches, ["torch"]) + + image_tensor = image_tensor.unsqueeze(0) + patches = torch.nn.functional.unfold(image_tensor, (patch_height, patch_width), stride=(patch_height, patch_width)) + patches = patches.reshape(image_tensor.size(0), image_tensor.size(1), patch_height, patch_width, -1) + patches = patches.permute(0, 4, 2, 3, 1).reshape( + image_tensor.size(2) // patch_height, + image_tensor.size(3) // patch_width, + image_tensor.size(1) * patch_height * patch_width, + ) + return patches.unsqueeze(0) + + +# Adapted from https://github.com/google-research/pix2struct/blob/0e1779af0f4db4b652c1d92b3bbd2550a7399123/pix2struct/preprocessing/preprocessing_utils.py#L106 +def render_text( + text: str, + text_size: int = 36, + text_color: str = "black", + background_color: str = "white", + left_padding: int = 5, + right_padding: int = 5, + top_padding: int = 5, + bottom_padding: int = 5, + font_bytes: Optional[bytes] = None, + font_path: Optional[str] = None, +) -> Image.Image: + """ + Render text. This script is entirely adapted from the original script that can be found here: + https://github.com/google-research/pix2struct/blob/main/pix2struct/preprocessing/preprocessing_utils.py + + Args: + text (`str`, *optional*, defaults to ): + Text to render. + text_size (`int`, *optional*, defaults to 36): + Size of the text. + text_color (`str`, *optional*, defaults to `"black"`): + Color of the text. + background_color (`str`, *optional*, defaults to `"white"`): + Color of the background. + left_padding (`int`, *optional*, defaults to 5): + Padding on the left. + right_padding (`int`, *optional*, defaults to 5): + Padding on the right. + top_padding (`int`, *optional*, defaults to 5): + Padding on the top. + bottom_padding (`int`, *optional*, defaults to 5): + Padding on the bottom. + font_bytes (`bytes`, *optional*): + Bytes of the font to use. If `None`, the default font will be used. + font_path (`str`, *optional*): + Path to the font to use. If `None`, the default font will be used. + """ + requires_backends(render_text, "vision") + # Add new lines so that each line is no more than 80 characters. + + wrapper = textwrap.TextWrapper(width=80) + lines = wrapper.wrap(text=text) + wrapped_text = "\n".join(lines) + + if font_bytes is not None and font_path is None: + font = io.BytesIO(font_bytes) + elif font_path is not None: + font = font_path + else: + font = hf_hub_download(DEFAULT_FONT_PATH, "Arial.TTF") + font = ImageFont.truetype(font, encoding="UTF-8", size=text_size) + + # Use a temporary canvas to determine the width and height in pixels when + # rendering the text. + temp_draw = ImageDraw.Draw(Image.new("RGB", (1, 1), background_color)) + _, _, text_width, text_height = temp_draw.textbbox((0, 0), wrapped_text, font) + + # Create the actual image with a bit of padding around the text. + image_width = text_width + left_padding + right_padding + image_height = text_height + top_padding + bottom_padding + image = Image.new("RGB", (image_width, image_height), background_color) + draw = ImageDraw.Draw(image) + draw.text(xy=(left_padding, top_padding), text=wrapped_text, fill=text_color, font=font) + return image + + +# Adapted from https://github.com/google-research/pix2struct/blob/0e1779af0f4db4b652c1d92b3bbd2550a7399123/pix2struct/preprocessing/preprocessing_utils.py#L87 +def render_header(image: np.ndarray, header: str, **kwargs): + """ + Renders the input text as a header on the input image. + + Args: + image (`np.ndarray`): + The image to render the header on. + header (`str`): + The header text. + data_format (`Union[ChannelDimension, str]`, *optional*): + The data format of the image. Can be either "ChannelDimension.channels_first" or + "ChannelDimension.channels_last". + + Returns: + `np.ndarray`: The image with the header rendered. + """ + requires_backends(render_header, "vision") + + # Convert to PIL image if necessary + image = to_pil_image(image) + + header_image = render_text(header, **kwargs) + new_width = max(header_image.width, image.width) + + new_height = int(image.height * (new_width / image.width)) + new_header_height = int(header_image.height * (new_width / header_image.width)) + + new_image = Image.new("RGB", (new_width, new_height + new_header_height), "white") + new_image.paste(header_image.resize((new_width, new_header_height)), (0, 0)) + new_image.paste(image.resize((new_width, new_height)), (0, new_header_height)) + + # Convert back to the original framework if necessary + new_image = to_numpy_array(new_image) + + if infer_channel_dimension_format(new_image) == ChannelDimension.LAST: + new_image = to_channel_dimension_format(new_image, ChannelDimension.LAST) + + return new_image + + +class Pix2StructImageProcessor(BaseImageProcessor): + r""" + Constructs a Pix2Struct image processor. + + Args: + do_convert_rgb (`bool`, *optional*, defaults to `True`): + Whether to convert the image to RGB. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` + method. According to Pix2Struct paper and code, the image is normalized with its own mean and standard + deviation. + patch_size (`Dict[str, int]`, *optional*, defaults to `{"height": 16, "width": 16}`): + The patch size to use for the image. According to Pix2Struct paper and code, the patch size is 16x16. + max_patches (`int`, *optional*, defaults to 2048): + The maximum number of patches to extract from the image as per the [Pix2Struct + paper](https://arxiv.org/pdf/2210.03347.pdf). + is_vqa (`bool`, *optional*, defaults to `False`): + Whether or not the image processor is for the VQA task. If `True` and `header_text` is passed in, text is + rendered onto the input images. + """ + + model_input_names = ["flattened_patches"] + + def __init__( + self, + do_convert_rgb: bool = True, + do_normalize: bool = True, + patch_size: Dict[str, int] = None, + max_patches: int = 2048, + is_vqa: bool = False, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.patch_size = patch_size if patch_size is not None else {"height": 16, "width": 16} + self.do_normalize = do_normalize + self.do_convert_rgb = do_convert_rgb + self.max_patches = max_patches + self.is_vqa = is_vqa + + def extract_flattened_patches(self, image: np.ndarray, max_patches: int, patch_size: dict, **kwargs) -> np.ndarray: + """ + Extract flattened patches from an image. + + Args: + image (`np.ndarray`): + Image to extract flattened patches from. + max_patches (`int`): + Maximum number of patches to extract. + patch_size (`dict`): + Dictionary containing the patch height and width. + + Returns: + result (`np.ndarray`): + A sequence of `max_patches` flattened patches. + """ + requires_backends(self.extract_flattened_patches, "torch") + + # convert to torch + image = to_channel_dimension_format(image, ChannelDimension.FIRST) + image = torch.from_numpy(image) + + patch_height, patch_width = patch_size["height"], patch_size["width"] + image_height, image_width = get_image_size(image) + + # maximize scale s.t. + scale = math.sqrt(max_patches * (patch_height / image_height) * (patch_width / image_width)) + num_feasible_rows = max(min(math.floor(scale * image_height / patch_height), max_patches), 1) + num_feasible_cols = max(min(math.floor(scale * image_width / patch_width), max_patches), 1) + resized_height = max(num_feasible_rows * patch_height, 1) + resized_width = max(num_feasible_cols * patch_width, 1) + + image = torch.nn.functional.interpolate( + image.unsqueeze(0), + size=(resized_height, resized_width), + mode="bilinear", + align_corners=False, + antialias=True, + ).squeeze(0) + + # [1, rows, columns, patch_height * patch_width * image_channels] + patches = torch_extract_patches(image, patch_height, patch_width) + + patches_shape = patches.shape + rows = patches_shape[1] + columns = patches_shape[2] + depth = patches_shape[3] + + # [rows * columns, patch_height * patch_width * image_channels] + patches = patches.reshape([rows * columns, depth]) + + # [rows * columns, 1] + row_ids = torch.arange(rows).reshape([rows, 1]).repeat(1, columns).reshape([rows * columns, 1]) + col_ids = torch.arange(columns).reshape([1, columns]).repeat(rows, 1).reshape([rows * columns, 1]) + + # Offset by 1 so the ids do not contain zeros, which represent padding. + row_ids += 1 + col_ids += 1 + + # Prepare additional patch features. + # [rows * columns, 1] + row_ids = row_ids.to(torch.float32) + col_ids = col_ids.to(torch.float32) + + # [rows * columns, 2 + patch_height * patch_width * image_channels] + result = torch.cat([row_ids, col_ids, patches], -1) + + # [max_patches, 2 + patch_height * patch_width * image_channels] + result = torch.nn.functional.pad(result, [0, 0, 0, max_patches - (rows * columns)]).float() + + result = to_numpy_array(result) + + return result + + def normalize( + self, image: np.ndarray, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + The image std is to mimic the tensorflow implementation of the `per_image_standardization`: + https://www.tensorflow.org/api_docs/python/tf/image/per_image_standardization + + Args: + image (`np.ndarray`): + Image to normalize. + """ + if image.dtype == np.uint8: + image = image.astype(np.float32) + + # take mean across the whole `image` + mean = np.mean(image) + std = np.std(image) + adjusted_stddev = max(std, 1.0 / math.sqrt(np.prod(image.shape))) + + return normalize(image, mean=mean, std=adjusted_stddev, **kwargs) + + def preprocess( + self, + images: ImageInput, + header_text: Optional[str] = None, + do_convert_rgb: bool = None, + do_normalize: Optional[bool] = None, + max_patches: Optional[int] = None, + patch_size: Optional[Dict[str, int]] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: ChannelDimension = ChannelDimension.FIRST, + **kwargs, + ) -> ImageInput: + """ + Preprocess an image or batch of images. The processor first computes the maximum possible number of + aspect-ratio preserving patches of size `patch_size` that can be extracted from the image. It then pads the + image with zeros to make the image respect the constraint of `max_patches`. Before extracting the patches the + images are standardized following the tensorflow implementation of `per_image_standardization` + (https://www.tensorflow.org/api_docs/python/tf/image/per_image_standardization). + + + Args: + images (`ImageInput`): + Image to preprocess. + header_text (`Union[List[str], str]`, *optional*): + Text to render as a header. Only has an effect if `image_processor.is_vqa` is `True`. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + max_patches (`int`, *optional*, defaults to `self.max_patches`): + Maximum number of patches to extract. + patch_size (`dict`, *optional*, defaults to `self.patch_size`): + Dictionary containing the patch height and width. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + """ + do_normalize = do_normalize if do_normalize is not None else self.do_normalize + do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb + patch_size = patch_size if patch_size is not None else self.patch_size + max_patches = max_patches if max_patches is not None else self.max_patches + is_vqa = self.is_vqa + + if kwargs.get("data_format", None) is not None: + raise ValueError("data_format is not an accepted input as the outputs are ") + + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + # PIL RGBA images are converted to RGB + if do_convert_rgb: + images = [convert_to_rgb(image) for image in images] + + # All transformations expect numpy arrays. + images = [to_numpy_array(image) for image in images] + + if is_vqa: + if header_text is None: + raise ValueError("A header text must be provided for VQA models.") + font_bytes = kwargs.pop("font_bytes", None) + font_path = kwargs.pop("font_path", None) + + if isinstance(header_text, str): + header_text = [header_text] * len(images) + + images = [ + render_header(image, header_text[i], font_bytes=font_bytes, font_path=font_path) + for i, image in enumerate(images) + ] + + if do_normalize: + images = [self.normalize(image=image) for image in images] + + # convert to torch tensor and permute + images = [ + self.extract_flattened_patches(image=image, max_patches=max_patches, patch_size=patch_size) + for image in images + ] + + # create attention mask in numpy + attention_masks = [(image.sum(axis=-1) != 0).astype(np.float32) for image in images] + + encoded_outputs = BatchFeature( + data={"flattened_patches": images, "attention_mask": attention_masks}, tensor_type=return_tensors + ) + + return encoded_outputs diff --git a/src/transformers/models/pix2struct/modeling_pix2struct.py b/src/transformers/models/pix2struct/modeling_pix2struct.py new file mode 100644 index 000000000000..1dcd22e0f44f --- /dev/null +++ b/src/transformers/models/pix2struct/modeling_pix2struct.py @@ -0,0 +1,1823 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. & Google team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Pix2Struct modeling file""" + +import copy +import math +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from torch.utils.checkpoint import checkpoint + +from ...activations import ACT2FN +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPooling, + CausalLMOutputWithCrossAttentions, + Seq2SeqLMOutput, + Seq2SeqModelOutput, +) +from ...modeling_utils import PreTrainedModel +from ...pytorch_utils import ALL_LAYERNORM_LAYERS +from ...utils import ( + DUMMY_INPUTS, + DUMMY_MASK, + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_torch_fx_proxy, + logging, + replace_return_docstrings, +) +from .configuration_pix2struct import Pix2StructConfig, Pix2StructTextConfig, Pix2StructVisionConfig + + +logger = logging.get_logger(__name__) + +# General docstring +_CONFIG_FOR_DOC = "Pix2StructConfig" + + +PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "google/pix2struct-textcaps-base", + "google/pix2struct-textcaps-large", + "google/pix2struct-base", + "google/pix2struct-large", + "google/pix2struct-ai2d-base", + "google/pix2struct-ai2d-large", + "google/pix2struct-widget-captioning-base", + "google/pix2struct-widget-captioning-large", + "google/pix2struct-screen2words-base", + "google/pix2struct-screen2words-large", + "google/pix2struct-docvqa-base", + "google/pix2struct-docvqa-large", + "google/pix2struct-ocrvqa-base", + "google/pix2struct-ocrvqa-large", + "google/pix2struct-chartqa-base", + "google/pix2struct-inforgraphics-vqa-base", + "google/pix2struct-inforgraphics-vqa-large", + # See all Pix2StructVision models at https://huggingface.co/models?filter=pix2struct +] + + +# Adapted from transformers.models.t5.modeling_t5.T5LayerNorm with T5->Pix2Struct +class Pix2StructLayerNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + Construct a layernorm module in the T5 style. No bias and no subtraction of mean. + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean + # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus varience is calculated + # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for + # half-precision inputs is done in fp32 + + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states + + +try: + from apex.normalization import FusedRMSNorm + + Pix2StructLayerNorm = FusedRMSNorm # noqa + + logger.info("Discovered apex.normalization.FusedRMSNorm - will use it instead of Pix2StructLayerNorm") +except ImportError: + # using the normal Pix2StructLayerNorm + pass +except Exception: + logger.warning("Discovered apex but it failed to load, falling back to Pix2StructLayerNorm") + pass + +ALL_LAYERNORM_LAYERS.append(Pix2StructLayerNorm) + + +class Pix2StructVisionEmbeddings(nn.Module): + r""" + Construct the embeddings from patch. In `Pix2Struct` the input is different from classic Vision-transformer models. + Here the input is a sequence of `seq_len` flattened patches that also combines padding patches (tokens). Each patch + is represented by a vector of `hidden_size` values. + """ + + def __init__(self, config: Pix2StructConfig) -> None: + super().__init__() + self.patch_projection = nn.Linear(config.patch_embed_hidden_size, config.hidden_size) + + self.row_embedder = nn.Embedding(config.seq_len, config.hidden_size) + self.column_embedder = nn.Embedding(config.seq_len, config.hidden_size) + + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, flattened_patches: torch.Tensor) -> torch.Tensor: + # the row and column indices are stored in the first and second position of the flattened_patches + # flattened_patches: `batch_size`, `seq_len`, `hidden_size` + 2 + row_indices = flattened_patches[:, :, 0].long() + col_indices = flattened_patches[:, :, 1].long() + + flattened_patches = flattened_patches[:, :, 2:] + + embeddings = self.patch_projection(flattened_patches) + row_embeddings = self.row_embedder(row_indices) + col_embeddings = self.column_embedder(col_indices) + + # sum all embeddings together + embeddings = embeddings + row_embeddings + col_embeddings + + embeddings = self.dropout(embeddings) + + return embeddings + + +class Pix2StructVisionAttention(nn.Module): + def __init__(self, config): + super().__init__() + self.hidden_size = config.hidden_size + self.key_value_proj_dim = config.d_kv + self.n_heads = config.num_attention_heads + self.dropout = config.attention_dropout + self.inner_dim = self.n_heads * self.key_value_proj_dim + + # Mesh TensorFlow initialization to avoid scaling before softmax + self.query = nn.Linear(self.hidden_size, self.inner_dim, bias=False) + self.key = nn.Linear(self.hidden_size, self.inner_dim, bias=False) + self.value = nn.Linear(self.hidden_size, self.inner_dim, bias=False) + self.output = nn.Linear(self.inner_dim, self.hidden_size, bias=False) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + output_attentions=False, + ): + """ + Self-attention block + """ + # Input is (batch_size, seq_length, dim) + # Mask is (batch_size, key_length) (non-causal) or (batch_size, key_length, key_length) + # past_key_value[0] is (batch_size, n_heads, q_len - 1, dim_per_head) + batch_size, seq_length = hidden_states.shape[:2] + + def to_projection_shape(states): + """projection""" + return states.contiguous().view(batch_size, -1, self.n_heads, self.key_value_proj_dim).transpose(1, 2) + + # get query states + # (batch_size, n_heads, seq_length, dim_per_head) + query_states = to_projection_shape(self.query(hidden_states)) + + # get key/value states + key_states = to_projection_shape(self.key(hidden_states)) + value_states = to_projection_shape(self.value(hidden_states)) + + # compute scores + # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9 + scores = torch.matmul(query_states, key_states.transpose(3, 2)) + + if position_bias is None: + position_bias = torch.zeros( + (1, self.n_heads, seq_length, seq_length), device=scores.device, dtype=scores.dtype + ) + if self.gradient_checkpointing and self.training: + position_bias.requires_grad = True + + if attention_mask is None: + attention_mask = torch.ones((batch_size, seq_length), device=scores.device, dtype=scores.dtype) + + if attention_mask.dim() == 2: + position_bias = position_bias + attention_mask[:, None, :, None].to(position_bias.device) + else: + # (batch_size, n_heads, seq_length, key_length) + position_bias = position_bias + attention_mask.to(position_bias.device) + position_bias = 1 - position_bias + + position_bias_masked = position_bias.masked_fill(position_bias == 1, torch.finfo(scores.dtype).min) + scores += position_bias_masked + scores = torch.max(scores, torch.tensor(torch.finfo(scores.dtype).min)) + + # (batch_size, n_heads, seq_length, key_length) + attn_weights = nn.functional.softmax(scores, dim=-1, dtype=torch.float32).type_as(scores) + + # (batch_size, n_heads, seq_length, key_length) + attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + # Mask heads if we want to + if layer_head_mask is not None: + attn_weights = attn_weights * layer_head_mask + + attn_output = torch.matmul(attn_weights, value_states) + + # (batch_size, seq_length, dim) + attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, -1, self.inner_dim) + + attn_output = self.output(attn_output) + + outputs = (attn_output,) + (position_bias,) + + if output_attentions: + outputs = outputs + (attn_weights,) + return outputs + + +# Copied from transformers.models.t5.modeling_t5.T5DenseGatedActDense with T5DenseGatedActDense->Pix2StructVisionMlp,T5Config->Pix2StructVisionConfig,config.d_model->config.hidden_size,dropout_rate->dropout_rate +class Pix2StructVisionMlp(nn.Module): + def __init__(self, config: Pix2StructVisionConfig): + super().__init__() + self.wi_0 = nn.Linear(config.hidden_size, config.d_ff, bias=False) + self.wi_1 = nn.Linear(config.hidden_size, config.d_ff, bias=False) + self.wo = nn.Linear(config.d_ff, config.hidden_size, bias=False) + self.dropout = nn.Dropout(config.dropout_rate) + self.act = ACT2FN[config.dense_act_fn] + + def forward(self, hidden_states): + hidden_gelu = self.act(self.wi_0(hidden_states)) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states = self.dropout(hidden_states) + + # To make 8bit quantization work for google/flan-t5-xxl, self.wo is kept in float32. + # See https://github.com/huggingface/transformers/issues/20287 + # we also make sure the weights are not in `int8` in case users will force `_keep_in_fp32_modules` to be `None`` + if ( + isinstance(self.wo.weight, torch.Tensor) + and hidden_states.dtype != self.wo.weight.dtype + and self.wo.weight.dtype != torch.int8 + ): + hidden_states = hidden_states.to(self.wo.weight.dtype) + + hidden_states = self.wo(hidden_states) + return hidden_states + + +class Pix2StructVisionLayer(nn.Module): + def __init__(self, config: Pix2StructConfig) -> None: + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = Pix2StructVisionAttention(config) + self.mlp = Pix2StructVisionMlp(config) + self.pre_mlp_layer_norm = Pix2StructLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pre_attention_layer_norm = Pix2StructLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Union[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor]]: + residual = hidden_states + + # in Pix2StructVision, layernorm is applied before self-attention + hidden_states = self.pre_attention_layer_norm(hidden_states) + + self_attention_outputs = self.attention( + hidden_states, + attention_mask=attention_mask, + layer_head_mask=head_mask, + output_attentions=output_attentions, + ) + attention_output = self_attention_outputs[0] + outputs = self_attention_outputs[1:] # add self attentions if we output attention weights + + # first residual connection + hidden_states = attention_output + residual + + # in Pix2StructVision, layernorm is also applied after self-attention + layer_output = self.pre_mlp_layer_norm(hidden_states) + layer_output = self.mlp(layer_output) + hidden_states # second residual connection + + outputs = (layer_output,) + outputs + + return outputs + + +class Pix2StructVisionEncoder(nn.Module): + def __init__(self, config: Pix2StructConfig) -> None: + super().__init__() + self.config = config + self.layer = nn.ModuleList([Pix2StructVisionLayer(config) for _ in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + ) -> Union[tuple, BaseModelOutput]: + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_head_mask = head_mask[i] if head_mask is not None else None + + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + ) + else: + layer_outputs = layer_module(hidden_states, attention_mask, layer_head_mask, output_attentions) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + +class Pix2StructPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = Pix2StructConfig + + @property + def dummy_inputs(self): + input_ids = torch.tensor(DUMMY_INPUTS) + input_mask = torch.tensor(DUMMY_MASK) + dummy_inputs = { + "decoder_input_ids": input_ids, + "input_ids": input_ids, + "decoder_attention_mask": input_mask, + } + return dummy_inputs + + def _init_weights(self, module): + """Initialize the weights""" + factor = self.config.initializer_factor # Used for testing weights initialization + if isinstance(module, Pix2StructLayerNorm): + module.weight.data.fill_(factor * 1.0) + elif isinstance(module, Pix2StructTextDenseGatedActDense): + hidden_size = ( + self.config.text_config.hidden_size + if isinstance(self.config, Pix2StructConfig) + else self.config.hidden_size + ) + d_ff = self.config.text_config.d_ff if isinstance(self.config, Pix2StructConfig) else self.config.d_ff + + module.wi_0.weight.data.normal_(mean=0.0, std=factor * ((hidden_size) ** -0.5)) + if hasattr(module.wi_0, "bias") and module.wi_0.bias is not None: + module.wi_0.bias.data.zero_() + module.wi_1.weight.data.normal_(mean=0.0, std=factor * ((hidden_size) ** -0.5)) + if hasattr(module.wi_1, "bias") and module.wi_1.bias is not None: + module.wi_1.bias.data.zero_() + module.wo.weight.data.normal_(mean=0.0, std=factor * ((d_ff) ** -0.5)) + if hasattr(module.wo, "bias") and module.wo.bias is not None: + module.wo.bias.data.zero_() + elif isinstance(module, Pix2StructTextAttention): + # Mesh TensorFlow attention initialization to avoid scaling before softmax + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/attention.py#L136 + hidden_size = ( + self.config.text_config.hidden_size + if isinstance(self.config, Pix2StructConfig) + else self.config.hidden_size + ) + key_value_proj_dim = ( + self.config.text_config.d_kv if isinstance(self.config, Pix2StructConfig) else self.config.hidden_size + ) + n_heads = ( + self.config.text_config.num_heads + if isinstance(self.config, Pix2StructConfig) + else self.config.num_heads + ) + + module.query.weight.data.normal_(mean=0.0, std=factor * ((hidden_size * key_value_proj_dim) ** -0.5)) + module.key.weight.data.normal_(mean=0.0, std=factor * (hidden_size**-0.5)) + module.value.weight.data.normal_(mean=0.0, std=factor * (hidden_size**-0.5)) + module.output.weight.data.normal_(mean=0.0, std=factor * ((n_heads * key_value_proj_dim) ** -0.5)) + if module.has_relative_attention_bias: + module.relative_attention_bias.weight.data.normal_(mean=0.0, std=factor * ((hidden_size) ** -0.5)) + elif isinstance(module, nn.Embedding): + hidden_size = ( + self.config.text_config.hidden_size + if isinstance(self.config, Pix2StructConfig) + else self.config.hidden_size + ) + + module.weight.data.normal_(mean=0.0, std=factor * ((hidden_size) ** -0.5)) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, Pix2StructTextModel): + hidden_size = ( + self.config.text_config.hidden_size + if isinstance(self.config, Pix2StructConfig) + else self.config.hidden_size + ) + + module.lm_head.weight.data.normal_(mean=0.0, std=factor * ((hidden_size) ** -0.5)) + elif isinstance(module, (nn.Linear, nn.Conv2d)): + # Upcast the input in `fp32` and cast it back to desired `dtype` to avoid + # `trunc_normal_cpu` not implemented in `half` issues + module.weight.data = nn.init.trunc_normal_( + module.weight.data.to(torch.float32), mean=0.0, std=self.config.initializer_range + ).to(module.weight.dtype) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, Pix2StructLayerNorm): + if module.weight is not None: + module.weight.data.fill_(1.0) + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + # Copied from transformers.models.t5.modeling_t5.T5PreTrainedModel._shift_right with T5->Pix2Struct + def _shift_right(self, input_ids): + decoder_start_token_id = self.config.decoder_start_token_id + pad_token_id = self.config.pad_token_id + + assert decoder_start_token_id is not None, ( + "self.model.config.decoder_start_token_id has to be defined. In Pix2Struct it is usually set to the pad_token_id." + " See Pix2Struct docs for more information" + ) + + # shift inputs to the right + if is_torch_fx_proxy(input_ids): + # Item assignment is not supported natively for proxies. + shifted_input_ids = torch.full(input_ids.shape[:-1] + (1,), decoder_start_token_id) + shifted_input_ids = torch.cat([shifted_input_ids, input_ids[..., :-1]], dim=-1) + else: + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() + shifted_input_ids[..., 0] = decoder_start_token_id + + assert pad_token_id is not None, "self.model.config.pad_token_id has to be defined." + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + return shifted_input_ids + + +PIX2STRUCT_VISION_START_DOCSTRING = r""" + This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it + as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and + behavior. + + Parameters: + config ([`Pix2StructConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +PIX2STRUCT_VISION_INPUTS_DOCSTRING = r""" + Args: + flattened_patches (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_channels x patch_height x patch_width)`): + Flattened and padded pixel values. These values can be obtained using [`AutoImageProcessor`]. See + [`Pix2StructVisionImageProcessor.__call__`] for details. Check the [original + paper](https://arxiv.org/abs/2210.03347) (figure 5) for more details. + + attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`: + + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare Pix2StructVision Model transformer outputting raw hidden-states without any specific head on top.", + PIX2STRUCT_VISION_START_DOCSTRING, +) +class Pix2StructVisionModel(Pix2StructPreTrainedModel): + config_class = Pix2StructVisionConfig + main_input_name = "flattened_patches" + supports_gradient_checkpointing = True + _no_split_modules = ["Pix2StructVisionLayer"] + + def __init__(self, config: Pix2StructConfig): + super().__init__(config) + self.config = config + + self.embeddings = Pix2StructVisionEmbeddings(config) + self.encoder = Pix2StructVisionEncoder(config) + + self.layernorm = Pix2StructLayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + # Initialize weights and apply final processing + self.post_init() + + def _set_gradient_checkpointing(self, module: Pix2StructVisionEncoder, value: bool = False) -> None: + if isinstance(module, Pix2StructVisionEncoder): + module.gradient_checkpointing = value + + def get_input_embeddings(self): + return self.embeddings.patch_projection + + def _prune_heads(self, heads_to_prune: Dict[int, List[int]]) -> None: + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + @add_start_docstrings_to_model_forward(PIX2STRUCT_VISION_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + def forward( + self, + flattened_patches: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + Example: + + ```python + >>> import requests + >>> from PIL import Image + >>> from transformers import AutoProcessor, Pix2StructVisionModel + + >>> image_processor = AutoProcessor.from_pretrained("google/pix2struct-textcaps-base") + >>> model = Pix2StructVisionModel.from_pretrained("google/pix2struct-textcaps-base") + + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = image_processor(images=image, return_tensors="pt") + >>> with torch.no_grad(): + ... outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + >>> list(last_hidden_states.shape) + [1, 2048, 768] + ``` + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if flattened_patches is None: + raise ValueError("You have to specify flattened_patches") + + if attention_mask is None: + # check where `flattened_patches` is not 0 + attention_mask = (flattened_patches.sum(dim=-1) != 0).float() + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) + + embedding_output = self.embeddings(flattened_patches) + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=attention_mask, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + sequence_output = self.layernorm(sequence_output) + + if not return_dict: + head_outputs = (sequence_output,) + return head_outputs + encoder_outputs[1:] + + return BaseModelOutput( + last_hidden_state=sequence_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +# Copied from transformers.models.t5.modeling_t5.T5DenseGatedActDense with T5->Pix2StructText,d_model->hidden_size +class Pix2StructTextDenseGatedActDense(nn.Module): + def __init__(self, config: Pix2StructTextConfig): + super().__init__() + self.wi_0 = nn.Linear(config.hidden_size, config.d_ff, bias=False) + self.wi_1 = nn.Linear(config.hidden_size, config.d_ff, bias=False) + self.wo = nn.Linear(config.d_ff, config.hidden_size, bias=False) + self.dropout = nn.Dropout(config.dropout_rate) + self.act = ACT2FN[config.dense_act_fn] + + def forward(self, hidden_states): + hidden_gelu = self.act(self.wi_0(hidden_states)) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states = self.dropout(hidden_states) + + # To make 8bit quantization work for google/flan-t5-xxl, self.wo is kept in float32. + # See https://github.com/huggingface/transformers/issues/20287 + # we also make sure the weights are not in `int8` in case users will force `_keep_in_fp32_modules` to be `None`` + if ( + isinstance(self.wo.weight, torch.Tensor) + and hidden_states.dtype != self.wo.weight.dtype + and self.wo.weight.dtype != torch.int8 + ): + hidden_states = hidden_states.to(self.wo.weight.dtype) + + hidden_states = self.wo(hidden_states) + return hidden_states + + +class Pix2StructTextLayerFF(nn.Module): + def __init__(self, config: Pix2StructTextConfig): + super().__init__() + self.DenseReluDense = Pix2StructTextDenseGatedActDense(config) + + self.layer_norm = Pix2StructLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + # Copied from transformers.models.t5.modeling_t5.T5LayerFF.forward + def forward(self, hidden_states): + forwarded_states = self.layer_norm(hidden_states) + forwarded_states = self.DenseReluDense(forwarded_states) + hidden_states = hidden_states + self.dropout(forwarded_states) + return hidden_states + + +class Pix2StructTextAttention(nn.Module): + def __init__(self, config: Pix2StructTextConfig, has_relative_attention_bias=False): + super().__init__() + self.has_relative_attention_bias = has_relative_attention_bias + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.relative_attention_max_distance = config.relative_attention_max_distance + self.hidden_size = config.hidden_size + self.key_value_proj_dim = config.d_kv + self.n_heads = config.num_heads + self.dropout = config.dropout_rate + self.inner_dim = self.n_heads * self.key_value_proj_dim + + # Mesh TensorFlow initialization to avoid scaling before softmax + self.query = nn.Linear(self.hidden_size, self.hidden_size, bias=False) + self.key = nn.Linear(self.hidden_size, self.hidden_size, bias=False) + self.value = nn.Linear(self.hidden_size, self.hidden_size, bias=False) + self.output = nn.Linear(self.hidden_size, self.hidden_size, bias=False) + + if self.has_relative_attention_bias: + self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads) + self.pruned_heads = set() + self.gradient_checkpointing = False + + @staticmethod + # Copied from transformers.models.t5.modeling_t5.T5Attention._relative_position_bucket + def _relative_position_bucket(relative_position, bidirectional=True, num_buckets=32, max_distance=128): + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + + Translate relative position to a bucket number for relative attention. The relative position is defined as + memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to + position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for + small absolute relative_position and larger buckets for larger absolute relative_positions. All relative + positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. + This should allow for more graceful generalization to longer sequences than the model has been trained on + + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + + Returns: + a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) + """ + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets += (relative_position > 0).to(torch.long) * num_buckets + relative_position = torch.abs(relative_position) + else: + relative_position = -torch.min(relative_position, torch.zeros_like(relative_position)) + # now relative_position is in the range [0, inf) + + # half of the buckets are for exact increments in positions + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + + # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance + relative_position_if_large = max_exact + ( + torch.log(relative_position.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(torch.long) + relative_position_if_large = torch.min( + relative_position_if_large, torch.full_like(relative_position_if_large, num_buckets - 1) + ) + + relative_buckets += torch.where(is_small, relative_position, relative_position_if_large) + return relative_buckets + + # Adapted from transformers.models.t5.modeling_t5.T5Attention.compute_bias + def compute_bias(self, query_length, key_length, device=None): + """Compute binned relative position bias""" + if device is None: + device = self.relative_attention_bias.weight.device + context_position = torch.arange(query_length, dtype=torch.long, device=device)[:, None] + memory_position = torch.arange(key_length, dtype=torch.long, device=device)[None, :] + relative_position = memory_position - context_position # shape (query_length, key_length) + relative_position_bucket = self._relative_position_bucket( + relative_position, # shape (query_length, key_length) + bidirectional=False, + num_buckets=self.relative_attention_num_buckets, + max_distance=self.relative_attention_max_distance, + ) + values = self.relative_attention_bias(relative_position_bucket) # shape (query_length, key_length, num_heads) + values = values.permute([2, 0, 1]).unsqueeze(0) # shape (1, num_heads, query_length, key_length) + return values + + def forward( + self, + hidden_states, + mask=None, + key_value_states=None, + position_bias=None, + past_key_value=None, + layer_head_mask=None, + query_length=None, + use_cache=False, + output_attentions=False, + ): + """ + Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states). + """ + # Input is (batch_size, seq_length, dim) + # Mask is (batch_size, key_length) (non-causal) or (batch_size, key_length, key_length) + # past_key_value[0] is (batch_size, n_heads, q_len - 1, dim_per_head) + batch_size, seq_length = hidden_states.shape[:2] + + real_seq_length = seq_length + + if past_key_value is not None: + if len(past_key_value) != 2: + raise ValueError( + f"past_key_value should have 2 past states: keys and values. Got { len(past_key_value)} past states" + ) + real_seq_length += past_key_value[0].shape[2] if query_length is None else query_length + + key_length = real_seq_length if key_value_states is None else key_value_states.shape[1] + + def to_projection_shape(states): + """projection""" + return states.contiguous().view(batch_size, -1, self.n_heads, self.key_value_proj_dim).transpose(1, 2) + + def project(hidden_states, proj_layer, key_value_states, past_key_value): + """projects hidden states correctly to key/query states""" + if key_value_states is None: + # self-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = to_projection_shape(proj_layer(hidden_states)) + elif past_key_value is None: + # cross-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = to_projection_shape(proj_layer(key_value_states)) + + if past_key_value is not None: + if key_value_states is None: + # self-attn + # (batch_size, n_heads, key_length, dim_per_head) + hidden_states = torch.cat([past_key_value, hidden_states], dim=2) + elif past_key_value.shape[2] != key_value_states.shape[1]: + # checking that the `sequence_length` of the `past_key_value` is the same as + # the provided `key_value_states` to support prefix tuning + # cross-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = to_projection_shape(proj_layer(key_value_states)) + else: + # cross-attn + hidden_states = past_key_value + return hidden_states + + # get query states + # (batch_size, n_heads, seq_length, dim_per_head) + query_states = to_projection_shape(self.query(hidden_states)) + + # get key/value states + key_states = project( + hidden_states, self.key, key_value_states, past_key_value[0] if past_key_value is not None else None + ) + value_states = project( + hidden_states, self.value, key_value_states, past_key_value[1] if past_key_value is not None else None + ) + + # compute scores + scores = torch.matmul( + query_states, key_states.transpose(3, 2) + ) # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9 + + if position_bias is None: + if not self.has_relative_attention_bias: + position_bias = torch.zeros( + (1, self.n_heads, real_seq_length, key_length), device=scores.device, dtype=scores.dtype + ) + if self.gradient_checkpointing and self.training: + position_bias.requires_grad = True + else: + position_bias = self.compute_bias(real_seq_length, key_length, device=scores.device) + + # if key and values are already calculated + # we want only the last query position bias + if past_key_value is not None: + position_bias = position_bias[:, :, -hidden_states.size(1) :, :] + + if mask is not None: + position_bias = position_bias + mask # (batch_size, n_heads, seq_length, key_length) + + if self.pruned_heads: + mask = torch.ones(position_bias.shape[1]) + mask[list(self.pruned_heads)] = 0 + position_bias_masked = position_bias[:, mask.bool()] + else: + position_bias_masked = position_bias + + scores += position_bias_masked + # (batch_size, n_heads, seq_length, key_length) + attn_weights = nn.functional.softmax(scores.float(), dim=-1).type_as(scores) + + # (batch_size, n_heads, seq_length, key_length) + attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + # Mask heads if we want to + if layer_head_mask is not None: + attn_weights = attn_weights * layer_head_mask + + attn_output = torch.matmul(attn_weights, value_states) + # (batch_size, seq_length, dim) + attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, -1, self.inner_dim) + + attn_output = self.output(attn_output) + + present_key_value_state = (key_states, value_states) if use_cache else None + outputs = (attn_output,) + (present_key_value_state,) + (position_bias,) + + if output_attentions: + outputs = outputs + (attn_weights,) + return outputs + + +# Copied from transformers.models.t5.modeling_t5.T5LayerSelfAttention with T5LayerNorm->Pix2StructLayerNorm,T5Attention->Pix2StructTextAttention,self.SelfAttention->self.attention,config.d_model->config.hidden_size +class Pix2StructTextLayerSelfAttention(nn.Module): + def __init__(self, config, has_relative_attention_bias=False): + super().__init__() + self.attention = Pix2StructTextAttention(config, has_relative_attention_bias=has_relative_attention_bias) + self.layer_norm = Pix2StructLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + past_key_value=None, + use_cache=False, + output_attentions=False, + ): + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.attention( + normed_hidden_states, + mask=attention_mask, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states = hidden_states + self.dropout(attention_output[0]) + outputs = (hidden_states,) + attention_output[1:] # add attentions if we output them + return outputs + + +# Copied from transformers.models.t5.modeling_t5.T5LayerCrossAttention with T5LayerNorm->Pix2StructLayerNorm,T5Attention->Pix2StructTextAttention,self.EncDecAttention->self.attention,config.d_model->config.hidden_size +class Pix2StructTextLayerCrossAttention(nn.Module): + def __init__(self, config): + super().__init__() + self.attention = Pix2StructTextAttention(config, has_relative_attention_bias=False) + self.layer_norm = Pix2StructLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward( + self, + hidden_states, + key_value_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + past_key_value=None, + use_cache=False, + query_length=None, + output_attentions=False, + ): + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.attention( + normed_hidden_states, + mask=attention_mask, + key_value_states=key_value_states, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + query_length=query_length, + output_attentions=output_attentions, + ) + layer_output = hidden_states + self.dropout(attention_output[0]) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class Pix2StructTextBlock(nn.Module): + def __init__(self, config, has_relative_attention_bias=False): + super().__init__() + + self.self_attention = Pix2StructTextLayerSelfAttention( + config, has_relative_attention_bias=has_relative_attention_bias + ) + + self.encoder_decoder_attention = Pix2StructTextLayerCrossAttention(config) + + self.mlp = Pix2StructTextLayerFF(config) + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + encoder_decoder_position_bias=None, + layer_head_mask=None, + cross_attn_layer_head_mask=None, + past_key_value=None, + use_cache=False, + output_attentions=False, + return_dict=True, + ): + if past_key_value is not None: + expected_num_past_key_values = 2 if encoder_hidden_states is None else 4 + + if len(past_key_value) != expected_num_past_key_values: + raise ValueError( + f"There should be {expected_num_past_key_values} past states. " + f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}" + f"Got {len(past_key_value)} past key / value states" + ) + + self_attn_past_key_value = past_key_value[:2] + cross_attn_past_key_value = past_key_value[2:] + else: + self_attn_past_key_value, cross_attn_past_key_value = None, None + + self_attention_outputs = self.self_attention( + hidden_states, + attention_mask=attention_mask, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=self_attn_past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states, present_key_value_state = self_attention_outputs[:2] + attention_outputs = self_attention_outputs[2:] # Keep self-attention outputs and relative position weights + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + do_cross_attention = encoder_hidden_states is not None + if do_cross_attention: + # the actual query length is unknown for cross attention + # if using past key value states. Need to inject it here + if present_key_value_state is not None: + query_length = present_key_value_state[0].shape[2] + else: + query_length = None + + cross_attention_outputs = self.encoder_decoder_attention( + hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, + layer_head_mask=cross_attn_layer_head_mask, + past_key_value=cross_attn_past_key_value, + query_length=query_length, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states = cross_attention_outputs[0] + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + # Combine self attn and cross attn key value states + if present_key_value_state is not None: + present_key_value_state = present_key_value_state + cross_attention_outputs[1] + + # Keep cross-attention outputs and relative position weights + attention_outputs = attention_outputs + cross_attention_outputs[2:] + + # Apply Feed Forward layer + hidden_states = self.mlp(hidden_states) + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states,) + + if use_cache: + outputs = outputs + (present_key_value_state,) + attention_outputs + else: + outputs = outputs + attention_outputs + + return outputs + + +PIX2STRUCT_START_DOCSTRING = r""" + + The Pix2Struct model was proposed in [Pix2Struct: Screenshot Parsing as Pretraining for Visual Language + Understanding](https://arxiv.org/abs/2210.03347) by Kenton Lee, Mandar Joshi, Iulia Turc, Hexiang Hu, Fangyu Liu, + Julian Eisenschlos, Urvashi Khandelwal, Peter Shaw, Ming-Wei Chang, Kristina Toutanova. It's an encoder decoder + transformer pre-trained in a image-to-text setting. + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config (Union[`Pix2StructConfig`, `Pix2StructTextConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +PIX2STRUCT_TEXT_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Pix2StructText is a model with relative position + embeddings so you should be able to pad the inputs on both the right and the left. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for detail. + + [What are input IDs?](../glossary#input-ids) + + To know more on how to prepare `input_ids` for pretraining take a look a [Pix2StructText + Training](./t5#training). + attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Pix2StructText uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [Pix2StructText + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at + the output of the last layer of the encoder. Used in the cross-attention of the decoder. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention layers. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more control over how to convert + `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + + If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value + of `inputs_embeds`. + + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +PIX2STRUCT_INPUTS_DOCSTRING = r""" + Args: + flattened_patches (`torch.FloatTensor` of shape `(batch_size, seq_length, hidden_size)`): + Flattened pixel patches. the `hidden_size` is obtained by the following formula: `hidden_size` = + `num_channels` * `patch_size` * `patch_size` + + The process of flattening the pixel patches is done by `Pix2StructProcessor`. + + attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + Pix2StructText uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + + To know more on how to prepare `decoder_input_ids` for pretraining take a look at [Pix2StructText + Training](./t5#training). + decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in + `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at + the output of the last layer of the encoder. Used in the cross-attention of the decoder. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention layers. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more control over how to convert + `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + + If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value + of `inputs_embeds`. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss for the decoder. + + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The standalone text decoder of Pix2Struct", + PIX2STRUCT_START_DOCSTRING, +) +class Pix2StructTextModel(Pix2StructPreTrainedModel): + config_class = Pix2StructTextConfig + _no_split_modules = ["Pix2StructTextBlock"] + supports_gradient_checkpointing = True + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (Pix2StructTextAttention, Pix2StructTextModel)): + module.gradient_checkpointing = value + + def __init__(self, config): + super().__init__(config) + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) + + self.layer = nn.ModuleList( + [Pix2StructTextBlock(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)] + ) + self.final_layer_norm = Pix2StructLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + self.gradient_checkpointing = False + + # Copied from transformers.models.t5.modeling_t5.T5PreTrainedModel._reorder_cache + def _reorder_cache(self, past_key_values, beam_idx): + # if decoder past is not included in output + # speedy decoding is disabled and no need to reorder + if past_key_values is None: + logger.warning("You might want to consider setting `use_cache=True` to speed up decoding") + return past_key_values + + reordered_decoder_past = () + for layer_past_states in past_key_values: + # get the correct batch idx from layer past batch dim + # batch dim of `past` is at 2nd position + reordered_layer_past_states = () + for layer_past_state in layer_past_states: + # need to set correct `past` for each of the four key / value states + reordered_layer_past_states = reordered_layer_past_states + ( + layer_past_state.index_select(0, beam_idx.to(layer_past_state.device)), + ) + + assert reordered_layer_past_states[0].shape == layer_past_states[0].shape + assert len(reordered_layer_past_states) == len(layer_past_states) + + reordered_decoder_past = reordered_decoder_past + (reordered_layer_past_states,) + return reordered_decoder_past + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, new_embeddings): + self.embed_tokens = new_embeddings + + @add_start_docstrings_to_model_forward(PIX2STRUCT_TEXT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + inputs_embeds=None, + head_mask=None, + cross_attn_head_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + labels=None, + return_dict=None, + **kwargs, + ): + r""" + Returns: + + Example: + + ```python + >>> from transformers import AutoProcessor, Pix2StructTextModel + + >>> processor = AutoProcessor.from_pretrained("google/pix2struct-textcaps-base") + >>> model = Pix2StructTextModel.from_pretrained("google/pix2struct-textcaps-base") + + >>> inputs = processor(text="Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + >>> loss = outputs.loss + ``` + """ + use_cache = use_cache if use_cache is not None else self.config.use_cache + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + if inputs_embeds is None: + assert self.embed_tokens is not None, "You have to initialize the model with valid token embeddings" + inputs_embeds = self.embed_tokens(input_ids) + + batch_size, seq_length = input_shape + + # required mask seq length can be calculated via length of past + mask_seq_length = past_key_values[0][0].shape[2] + seq_length if past_key_values is not None else seq_length + + if attention_mask is None: + attention_mask = torch.ones(batch_size, mask_seq_length, device=inputs_embeds.device) + if encoder_attention_mask is None and encoder_hidden_states is not None: + encoder_seq_length = encoder_hidden_states.shape[1] + encoder_attention_mask = torch.ones( + batch_size, encoder_seq_length, device=inputs_embeds.device, dtype=torch.long + ) + + # initialize past_key_values with `None` if past does not exist + if past_key_values is None: + past_key_values = [None] * len(self.layer) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=inputs_embeds.device) + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + head_mask = self.get_head_mask(head_mask, self.config.num_layers) + cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers) + present_key_value_states = () if use_cache else None + all_hidden_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + all_cross_attentions = () if (output_attentions) else None + position_bias = None + encoder_decoder_position_bias = None + + hidden_states = self.dropout(inputs_embeds) + + for i, (layer_module, past_key_value) in enumerate(zip(self.layer, past_key_values)): + layer_head_mask = head_mask[i] + cross_attn_layer_head_mask = cross_attn_head_mask[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + def create_custom_forward(module): + def custom_forward(*inputs): + return tuple(module(*inputs, use_cache, output_attentions)) + + return custom_forward + + layer_outputs = checkpoint( + create_custom_forward(layer_module), + hidden_states, + extended_attention_mask, + position_bias, + encoder_hidden_states, + encoder_extended_attention_mask, + encoder_decoder_position_bias, + layer_head_mask, + cross_attn_layer_head_mask, + None, # past_key_value is always None with gradient checkpointing + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask=extended_attention_mask, + position_bias=position_bias, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + encoder_decoder_position_bias=encoder_decoder_position_bias, + layer_head_mask=layer_head_mask, + cross_attn_layer_head_mask=cross_attn_layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + + # layer_outputs is a tuple with: + # hidden-states, key-value-states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights) + if use_cache is False: + layer_outputs = layer_outputs[:1] + (None,) + layer_outputs[1:] + + hidden_states, present_key_value_state = layer_outputs[:2] + + # We share the position biases between the layers - the first layer store them + # layer_outputs = hidden-states, key-value-states (self-attention position bias), (self-attention weights), + # (cross-attention position bias), (cross-attention weights) + position_bias = layer_outputs[2] + if encoder_hidden_states is not None: + encoder_decoder_position_bias = layer_outputs[4 if output_attentions else 3] + # append next layer key value states + if use_cache: + present_key_value_states = present_key_value_states + (present_key_value_state,) + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[2],) + all_cross_attentions = all_cross_attentions + (layer_outputs[3],) + + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.dropout(hidden_states) + + logits = self.lm_head(hidden_states) + + # Add last layer + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + loss = None + if labels is not None: + loss_fct = nn.CrossEntropyLoss(ignore_index=-100, reduction="mean", label_smoothing=0.1) + masked_labels = labels.masked_fill(labels == self.config.pad_token_id, -100) + + loss = loss_fct(logits.contiguous().view(-1, logits.size(-1)), masked_labels.contiguous().view(-1)) + + if not return_dict: + return tuple( + v + for v in [ + loss, + logits, + present_key_value_states, + all_hidden_states, + all_attentions, + all_cross_attentions, + ] + if v is not None + ) + return CausalLMOutputWithCrossAttentions( + loss=loss, + logits=logits, + past_key_values=present_key_value_states, + hidden_states=all_hidden_states, + attentions=all_attentions, + cross_attentions=all_cross_attentions, + ) + + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attention_mask=None, **model_kwargs): + input_shape = input_ids.shape + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # cut decoder_input_ids if past_key_values is used + if past_key_values is not None: + input_ids = input_ids[:, -1:] + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "past_key_values": past_key_values, + "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None), + "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None), + "is_decoder": True, + } + + +@add_start_docstrings( + "A conditional generation model with a language modeling head. Can be used for sequence generation tasks.", + PIX2STRUCT_START_DOCSTRING, +) +class Pix2StructForConditionalGeneration(Pix2StructPreTrainedModel): + config_class = Pix2StructConfig + main_input_name = "flattened_patches" + + _keys_to_ignore_on_load_missing = [ + r"encoder.embed_tokens.weight", + r"decoder.embed_tokens.weight", + ] + _keys_to_ignore_on_load_unexpected = [ + r"decoder.layer.0.layer.1.EncDecAttention.relative_attention_bias.weight", + ] + + def __init__(self, config: Pix2StructConfig): + super().__init__(config) + encoder_config = copy.deepcopy(config.vision_config) + self.encoder = Pix2StructVisionModel(encoder_config) + + decoder_config = copy.deepcopy(config.text_config) + self.decoder_start_token_id = decoder_config.pad_token_id + self.decoder_eos_token_ids = decoder_config.eos_token_id + self.decoder = Pix2StructTextModel(decoder_config) + + self.is_vqa = config.is_vqa + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, new_embeddings): + self.shared = new_embeddings + self.decoder.set_input_embeddings(new_embeddings) + + def get_decoder(self): + return self.decoder + + def get_encoder(self): + return self.encoder + + @add_start_docstrings_to_model_forward(PIX2STRUCT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + flattened_patches: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.BoolTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + decoder_head_mask: Optional[torch.FloatTensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + labels: Optional[torch.LongTensor] = None, + decoder_inputs_embeds: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.FloatTensor], Seq2SeqModelOutput]: + r""" + Returns: + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Pix2StructForConditionalGeneration + + >>> processor = AutoProcessor.from_pretrained("google/pix2struct-textcaps-base") + >>> model = Pix2StructForConditionalGeneration.from_pretrained("google/pix2struct-textcaps-base") + + >>> labels = "A stop sign is on the street corner." + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, text=labels, return_tensors="pt", add_special_tokens=True) + + >>> # forward pass + >>> outputs = model(**inputs) + >>> last_hidden_states = outputs.loss + ```""" + use_cache = use_cache if use_cache is not None else self.config.text_config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # Encode if needed (training, first prediction pass) + if encoder_outputs is None: + encoder_outputs = self.encoder( + flattened_patches=flattened_patches, + attention_mask=attention_mask, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = encoder_outputs[0] + + if labels is not None and decoder_input_ids is None and decoder_inputs_embeds is None: + # get decoder inputs from shifting lm labels to the right + decoder_input_ids = self._shift_right(labels) + decoder_attention_mask = ( + decoder_attention_mask + if decoder_attention_mask is not None + else decoder_input_ids.ne(self.config.pad_token_id).float() + ) + # Always attend to the first token + decoder_attention_mask[:, 0] = 1 + + # Decode + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + inputs_embeds=decoder_inputs_embeds, + past_key_values=past_key_values, + encoder_hidden_states=hidden_states, + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + labels=labels, + return_dict=return_dict, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return Seq2SeqLMOutput( + loss=decoder_outputs.loss, + logits=decoder_outputs.logits, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + @torch.no_grad() + def generate( + self, + flattened_patches: torch.FloatTensor, + decoder_input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + **generate_kwargs, + ): + r""" + Returns: + + Example: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import AutoProcessor, Pix2StructForConditionalGeneration + + >>> processor = AutoProcessor.from_pretrained("google/pix2struct-textcaps-base") + >>> model = Pix2StructForConditionalGeneration.from_pretrained("google/pix2struct-textcaps-base") + + >>> conditional_text = "A stop sign" + >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, text=conditional_text, return_tensors="pt", add_special_tokens=True) + + >>> # forward pass + >>> outputs = model.generate(**inputs) + >>> print(processor.batch_decode(outputs, skip_special_tokens=True)) + ['A stop sign the street with a sign that says yes'] + ```""" + batch_size, _, _ = flattened_patches.shape + + vision_outputs = self.encoder(flattened_patches=flattened_patches, attention_mask=attention_mask) + + image_embeds = vision_outputs[0] + + if isinstance(decoder_input_ids, torch.Tensor): + # check if the first element of `input_ids` is equal to `decoder_input_ids`: + if (decoder_input_ids[:, 0] != self.decoder_start_token_id).all().item(): + # add `decoder_input_ids` as first token to `input_ids` + decoder_input_ids = torch.cat( + [ + torch.ones((decoder_input_ids.shape[0], 1), dtype=torch.long, device=decoder_input_ids.device) + * self.decoder_start_token_id, + decoder_input_ids, + ], + dim=-1, + ) + + if decoder_attention_mask is not None: + decoder_attention_mask = torch.cat( + [ + torch.ones( + (decoder_attention_mask.shape[0], 1), + dtype=torch.long, + device=decoder_attention_mask.device, + ), + decoder_attention_mask, + ], + dim=-1, + ) + elif decoder_input_ids is None: + decoder_input_ids = ( + torch.LongTensor([[self.decoder_start_token_id]]).repeat(batch_size, 1).to(image_embeds.device) + ) + + if decoder_attention_mask is None: + decoder_attention_mask = torch.ones_like(decoder_input_ids).to(image_embeds.device) + + outputs = self.decoder.generate( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=attention_mask, + **generate_kwargs, + ) + + return outputs diff --git a/src/transformers/models/pix2struct/processing_pix2struct.py b/src/transformers/models/pix2struct/processing_pix2struct.py new file mode 100644 index 000000000000..eaa9f0dc4247 --- /dev/null +++ b/src/transformers/models/pix2struct/processing_pix2struct.py @@ -0,0 +1,162 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Processor class for Pix2Struct. +""" + +from typing import List, Optional, Union + +from ...processing_utils import ProcessorMixin +from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy +from ...utils import TensorType + + +class Pix2StructProcessor(ProcessorMixin): + r""" + Constructs a PIX2STRUCT processor which wraps a BERT tokenizer and PIX2STRUCT image processor into a single + processor. + + [`Pix2StructProcessor`] offers all the functionalities of [`Pix2StructImageProcessor`] and [`T5TokenizerFast`]. See + the docstring of [`~Pix2StructProcessor.__call__`] and [`~Pix2StructProcessor.decode`] for more information. + + Args: + image_processor (`Pix2StructImageProcessor`): + An instance of [`Pix2StructImageProcessor`]. The image processor is a required input. + tokenizer (Union[`T5TokenizerFast`, `T5Tokenizer`]): + An instance of ['T5TokenizerFast`] or ['T5Tokenizer`]. The tokenizer is a required input. + """ + attributes = ["image_processor", "tokenizer"] + image_processor_class = "Pix2StructImageProcessor" + tokenizer_class = ("T5Tokenizer", "T5TokenizerFast") + + def __init__(self, image_processor, tokenizer): + tokenizer.return_token_type_ids = False + super().__init__(image_processor, tokenizer) + + def __call__( + self, + images=None, + text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None, + add_special_tokens: bool = False, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TruncationStrategy] = None, + max_length: Optional[int] = None, + max_patches: Optional[int] = 2048, + stride: int = 0, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + return_overflowing_tokens: bool = False, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_token_type_ids: bool = False, + return_length: bool = False, + verbose: bool = True, + return_tensors: Optional[Union[str, TensorType]] = None, + **kwargs, + ) -> BatchEncoding: + """ + This method uses [`Pix2StructImageProcessor.preprocess`] method to prepare image(s) for the model, and + [`T5TokenizerFast.__call__`] to prepare text for the model. + + Please refer to the docstring of the above two methods for more information. + """ + if images is None and text is None: + raise ValueError("You have to specify either images or text.") + + # Get only text + if images is None and not self.image_processor.is_vqa: + self.current_processor = self.tokenizer + text_encoding = self.tokenizer( + text=text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + stride=stride, + pad_to_multiple_of=pad_to_multiple_of, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_token_type_ids=return_token_type_ids, + return_length=return_length, + verbose=verbose, + return_tensors=return_tensors, + **kwargs, + ) + return text_encoding + + if not self.image_processor.is_vqa: + # add pixel_values + encoding_image_processor = self.image_processor( + images, return_tensors=return_tensors, max_patches=max_patches, **kwargs + ) + else: + # add pixel_values and bbox + encoding_image_processor = self.image_processor( + images, return_tensors=return_tensors, max_patches=max_patches, header_text=text, **kwargs + ) + + if text is not None and not self.image_processor.is_vqa: + text_encoding = self.tokenizer( + text=text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + stride=stride, + pad_to_multiple_of=pad_to_multiple_of, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_token_type_ids=return_token_type_ids, + return_length=return_length, + verbose=verbose, + return_tensors=return_tensors, + **kwargs, + ) + + if "attention_mask" in text_encoding: + text_encoding["decoder_attention_mask"] = text_encoding.pop("attention_mask") + if "input_ids" in text_encoding: + text_encoding["decoder_input_ids"] = text_encoding.pop("input_ids") + else: + text_encoding = None + + if text_encoding is not None: + encoding_image_processor.update(text_encoding) + + return encoding_image_processor + + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to Pix2StructTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. + Please refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to Pix2StructTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please + refer to the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + tokenizer_input_names = self.tokenizer.model_input_names + image_processor_input_names = self.image_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index a80af49e2784..b062a3bec4d9 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -5049,6 +5049,37 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class Pix2StructForConditionalGeneration(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class Pix2StructPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class Pix2StructTextModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class Pix2StructVisionModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + PLBART_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/src/transformers/utils/dummy_vision_objects.py b/src/transformers/utils/dummy_vision_objects.py index 5960f1e4b47e..ea83c6b920e1 100644 --- a/src/transformers/utils/dummy_vision_objects.py +++ b/src/transformers/utils/dummy_vision_objects.py @@ -387,6 +387,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["vision"]) +class Pix2StructImageProcessor(metaclass=DummyObject): + _backends = ["vision"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["vision"]) + + class PoolFormerFeatureExtractor(metaclass=DummyObject): _backends = ["vision"] diff --git a/tests/models/pix2struct/__init__.py b/tests/models/pix2struct/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/pix2struct/test_image_processing_pix2struct.py b/tests/models/pix2struct/test_image_processing_pix2struct.py new file mode 100644 index 000000000000..e13e38b71c02 --- /dev/null +++ b/tests/models/pix2struct/test_image_processing_pix2struct.py @@ -0,0 +1,289 @@ +# coding=utf-8 +# Copyright 2023 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np +import requests + +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import is_torch_available, is_vision_available + +from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs + + +if is_torch_available(): + import torch + +if is_vision_available(): + from PIL import Image + + from transformers import Pix2StructImageProcessor + + +class Pix2StructImageProcessingTester(unittest.TestCase): + def __init__( + self, + parent, + batch_size=7, + num_channels=3, + image_size=18, + min_resolution=30, + max_resolution=400, + size=None, + do_normalize=True, + do_convert_rgb=True, + patch_size=None, + ): + size = size if size is not None else {"height": 20, "width": 20} + self.parent = parent + self.batch_size = batch_size + self.num_channels = num_channels + self.image_size = image_size + self.min_resolution = min_resolution + self.max_resolution = max_resolution + self.size = size + self.do_normalize = do_normalize + self.do_convert_rgb = do_convert_rgb + self.max_patches = [512, 1024, 2048, 4096] + self.patch_size = patch_size if patch_size is not None else {"height": 16, "width": 16} + + def prepare_image_processor_dict(self): + return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb} + + def prepare_dummy_image(self): + img_url = "https://www.ilankelman.org/stopsigns/australia.jpg" + raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB") + return raw_image + + +@require_torch +@require_vision +class Pix2StructImageProcessingTest(ImageProcessingSavingTestMixin, unittest.TestCase): + image_processing_class = Pix2StructImageProcessor if is_vision_available() else None + + def setUp(self): + self.image_processor_tester = Pix2StructImageProcessingTester(self) + + @property + def image_processor_dict(self): + return self.image_processor_tester.prepare_image_processor_dict() + + def test_image_processor_properties(self): + image_processor = self.image_processing_class(**self.image_processor_dict) + self.assertTrue(hasattr(image_processor, "do_normalize")) + self.assertTrue(hasattr(image_processor, "do_convert_rgb")) + + def test_expected_patches(self): + dummy_image = self.image_processor_tester.prepare_dummy_image() + + image_processor = self.image_processing_class(**self.image_processor_dict) + max_patch = 2048 + + inputs = image_processor(dummy_image, return_tensors="pt", max_patches=max_patch) + self.assertTrue(torch.allclose(inputs.flattened_patches.mean(), torch.tensor(0.0606), atol=1e-3, rtol=1e-3)) + + def test_call_pil(self): + # Initialize image_processor + image_processor = self.image_processing_class(**self.image_processor_dict) + # create random PIL images + image_inputs = prepare_image_inputs(self.image_processor_tester, equal_resolution=False) + for image in image_inputs: + self.assertIsInstance(image, Image.Image) + + # Test not batched input + expected_hidden_dim = ( + (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"]) + * self.image_processor_tester.num_channels + ) + 2 + + for max_patch in self.image_processor_tester.max_patches: + # Test not batched input + encoded_images = image_processor( + image_inputs[0], return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (1, max_patch, expected_hidden_dim), + ) + + # Test batched + encoded_images = image_processor( + image_inputs, return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim), + ) + + def test_call_vqa(self): + # Initialize image_processor + image_processor = self.image_processing_class(**self.image_processor_dict) + # create random PIL images + image_inputs = prepare_image_inputs(self.image_processor_tester, equal_resolution=False) + for image in image_inputs: + self.assertIsInstance(image, Image.Image) + + # Test not batched input + expected_hidden_dim = ( + (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"]) + * self.image_processor_tester.num_channels + ) + 2 + + image_processor.is_vqa = True + + for max_patch in self.image_processor_tester.max_patches: + # Test not batched input + with self.assertRaises(ValueError): + encoded_images = image_processor( + image_inputs[0], return_tensors="pt", max_patches=max_patch + ).flattened_patches + + dummy_text = "Hello" + + encoded_images = image_processor( + image_inputs[0], return_tensors="pt", max_patches=max_patch, header_text=dummy_text + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (1, max_patch, expected_hidden_dim), + ) + + # Test batched + encoded_images = image_processor( + image_inputs, return_tensors="pt", max_patches=max_patch, header_text=dummy_text + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim), + ) + + def test_call_numpy(self): + # Initialize image_processor + image_processor = self.image_processing_class(**self.image_processor_dict) + # create random numpy tensors + image_inputs = prepare_image_inputs(self.image_processor_tester, equal_resolution=False, numpify=True) + for image in image_inputs: + self.assertIsInstance(image, np.ndarray) + + expected_hidden_dim = ( + (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"]) + * self.image_processor_tester.num_channels + ) + 2 + + for max_patch in self.image_processor_tester.max_patches: + # Test not batched input + encoded_images = image_processor( + image_inputs[0], return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (1, max_patch, expected_hidden_dim), + ) + + # Test batched + encoded_images = image_processor( + image_inputs, return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim), + ) + + def test_call_pytorch(self): + # Initialize image_processor + image_processor = self.image_processing_class(**self.image_processor_dict) + # create random PyTorch tensors + image_inputs = prepare_image_inputs(self.image_processor_tester, equal_resolution=False, torchify=True) + for image in image_inputs: + self.assertIsInstance(image, torch.Tensor) + + # Test not batched input + expected_hidden_dim = ( + (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"]) + * self.image_processor_tester.num_channels + ) + 2 + + for max_patch in self.image_processor_tester.max_patches: + # Test not batched input + encoded_images = image_processor( + image_inputs[0], return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (1, max_patch, expected_hidden_dim), + ) + + # Test batched + encoded_images = image_processor( + image_inputs, return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim), + ) + + +@require_torch +@require_vision +class Pix2StructImageProcessingTestFourChannels(ImageProcessingSavingTestMixin, unittest.TestCase): + image_processing_class = Pix2StructImageProcessor if is_vision_available() else None + + def setUp(self): + self.image_processor_tester = Pix2StructImageProcessingTester(self, num_channels=4) + self.expected_encoded_image_num_channels = 3 + + @property + def image_processor_dict(self): + return self.image_processor_tester.prepare_image_processor_dict() + + def test_image_processor_properties(self): + image_processor = self.image_processing_class(**self.image_processor_dict) + self.assertTrue(hasattr(image_processor, "do_normalize")) + self.assertTrue(hasattr(image_processor, "do_convert_rgb")) + + def test_call_pil_four_channels(self): + # Initialize image_processor + image_processor = self.image_processing_class(**self.image_processor_dict) + # create random PIL images + image_inputs = prepare_image_inputs(self.image_processor_tester, equal_resolution=False) + for image in image_inputs: + self.assertIsInstance(image, Image.Image) + + # Test not batched input + expected_hidden_dim = ( + (self.image_processor_tester.patch_size["height"] * self.image_processor_tester.patch_size["width"]) + * (self.image_processor_tester.num_channels - 1) + ) + 2 + + for max_patch in self.image_processor_tester.max_patches: + # Test not batched input + encoded_images = image_processor( + image_inputs[0], return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (1, max_patch, expected_hidden_dim), + ) + + # Test batched + encoded_images = image_processor( + image_inputs, return_tensors="pt", max_patches=max_patch + ).flattened_patches + self.assertEqual( + encoded_images.shape, + (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim), + ) diff --git a/tests/models/pix2struct/test_modeling_pix2struct.py b/tests/models/pix2struct/test_modeling_pix2struct.py new file mode 100644 index 000000000000..3896f8d84584 --- /dev/null +++ b/tests/models/pix2struct/test_modeling_pix2struct.py @@ -0,0 +1,711 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch Pix2Struct model. """ + + +import inspect +import os +import tempfile +import unittest + +import numpy as np +import requests + +from transformers import Pix2StructConfig, Pix2StructTextConfig, Pix2StructVisionConfig +from transformers.testing_utils import require_torch, require_vision, slow, torch_device +from transformers.utils import is_torch_available, is_vision_available + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ( + ModelTesterMixin, + _config_zero_init, + floats_tensor, + ids_tensor, + random_attention_mask, +) + + +if is_torch_available(): + import torch + from torch import nn + + from transformers import ( + Pix2StructForConditionalGeneration, + Pix2StructProcessor, + Pix2StructTextModel, + Pix2StructVisionModel, + ) + from transformers.models.pix2struct.modeling_pix2struct import PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST + + +if is_vision_available(): + from PIL import Image + + +class Pix2StructVisionModelTester: + def __init__( + self, + parent, + batch_size=12, + image_size=30, + patch_size=2, + num_channels=3, + is_training=True, + hidden_size=12, + patch_embed_hidden_size=12, + projection_dim=32, + max_patches=64, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + dropout=0.1, + attention_dropout=0.1, + initializer_range=1e-10, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.image_size = image_size + self.patch_embed_hidden_size = patch_embed_hidden_size + self.patch_size = patch_size + self.num_channels = num_channels + self.is_training = is_training + self.hidden_size = hidden_size + self.max_patches = max_patches + self.seq_length = self.max_patches + self.patch_proj_dim = ((patch_size**2) * num_channels) + 2 + + self.projection_dim = projection_dim + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.dropout = dropout + self.attention_dropout = attention_dropout + self.initializer_range = initializer_range + self.scope = scope + + def prepare_config_and_inputs(self): + flattened_patches = floats_tensor([self.batch_size, self.max_patches, self.patch_proj_dim]) + config = self.get_config() + + return config, flattened_patches + + def get_config(self): + return Pix2StructVisionConfig( + image_size=self.image_size, + patch_size=self.patch_size, + num_channels=self.num_channels, + hidden_size=self.hidden_size, + projection_dim=self.projection_dim, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + dropout=self.dropout, + attention_dropout=self.attention_dropout, + initializer_range=self.initializer_range, + patch_embed_hidden_size=self.patch_embed_hidden_size, + ) + + def create_and_check_model(self, config, flattened_patches): + model = Pix2StructVisionModel(config=config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + result = model(flattened_patches) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, flattened_patches = config_and_inputs + inputs_dict = { + "flattened_patches": flattened_patches, + "attention_mask": torch.randint(0, 2, (self.batch_size, self.max_patches)), + } + return config, inputs_dict + + +@require_torch +class Pix2StructVisionModelTest(ModelTesterMixin, unittest.TestCase): + """ + Here we also overwrite some of the tests of test_modeling_common.py, as Pix2Struct does not use input_ids, inputs_embeds, + attention_mask and seq_length. + """ + + all_model_classes = (Pix2StructVisionModel,) if is_torch_available() else () + fx_compatible = False + test_pruning = False + test_resize_embeddings = False + test_head_masking = False + + def setUp(self): + self.model_tester = Pix2StructVisionModelTester(self) + self.config_tester = ConfigTester( + self, config_class=Pix2StructVisionConfig, has_text_modality=False, hidden_size=37 + ) + + def test_config(self): + self.config_tester.run_common_tests() + + @unittest.skip(reason="Pix2StructVision does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + def test_model_common_attributes(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + self.assertIsInstance(model.get_input_embeddings(), (nn.Module)) + x = model.get_output_embeddings() + self.assertTrue(x is None or isinstance(x, nn.Linear)) + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["flattened_patches"] + self.assertListEqual(arg_names[:1], expected_arg_names) + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + @unittest.skip(reason="Training is tested directly on `Pix2StructTextImageModelTest`") + def test_training(self): + pass + + @unittest.skip(reason="Training is tested directly on `Pix2StructTextImageModelTest`") + def test_training_gradient_checkpointing(self): + pass + + @unittest.skip(reason="Training is tested directly on `Pix2StructTextImageModelTest`") + def test_retain_grad_hidden_states_attentions(self): + pass + + @unittest.skip(reason="Pix2StructVisionModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_from_base(self): + pass + + @unittest.skip(reason="Pix2StructVisionModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_to_base(self): + pass + + @slow + def test_model_from_pretrained(self): + for model_name in PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = Pix2StructVisionModel.from_pretrained(model_name) + self.assertIsNotNone(model) + + +class Pix2StructTextModelTester: + def __init__( + self, + parent, + batch_size=12, + seq_length=7, + is_training=True, + use_input_mask=True, + use_labels=True, + vocab_size=99, + hidden_size=12, + projection_dim=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + dropout=0.1, + attention_dropout=0.1, + max_position_embeddings=512, + initializer_range=0.02, + bos_token_id=0, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_labels = use_labels + self.d_kv = hidden_size // num_attention_heads + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.projection_dim = projection_dim + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.dropout = dropout + self.attention_dropout = attention_dropout + self.max_position_embeddings = max_position_embeddings + self.initializer_range = initializer_range + self.scope = scope + self.bos_token_id = bos_token_id + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = random_attention_mask([self.batch_size, self.seq_length]) + + if input_mask is not None: + batch_size, seq_length = input_mask.shape + rnd_start_indices = np.random.randint(1, seq_length - 1, size=(batch_size,)) + for batch_idx, start_index in enumerate(rnd_start_indices): + input_mask[batch_idx, :start_index] = 1 + input_mask[batch_idx, start_index:] = 0 + + config = self.get_config() + + return config, input_ids, input_mask + + def get_config(self): + return Pix2StructTextConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + projection_dim=self.projection_dim, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + dropout=self.dropout, + attention_dropout=self.attention_dropout, + max_position_embeddings=self.max_position_embeddings, + initializer_range=self.initializer_range, + bos_token_id=self.bos_token_id, + d_kv=self.d_kv, + ) + + def create_and_check_model(self, config, input_ids, input_mask): + model = Pix2StructTextModel(config=config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + result = model(input_ids, attention_mask=input_mask) + result = model(input_ids) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, input_ids, input_mask = config_and_inputs + inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask} + return config, inputs_dict + + +@require_torch +class Pix2StructTextModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = (Pix2StructTextModel,) if is_torch_available() else () + fx_compatible = False + test_pruning = False + test_head_masking = False + + def setUp(self): + self.model_tester = Pix2StructTextModelTester(self) + self.config_tester = ConfigTester(self, config_class=Pix2StructTextConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + @unittest.skip(reason="Training is tested directly on `Pix2StructTextImageModelTest`") + def test_training(self): + pass + + @unittest.skip(reason="Training is tested directly on `Pix2StructTextImageModelTest`") + def test_training_gradient_checkpointing(self): + pass + + @unittest.skip(reason="Pix2Struct does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="Pix2StructTextModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_from_base(self): + pass + + @unittest.skip(reason="Pix2StructTextModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_to_base(self): + pass + + @slow + def test_model_from_pretrained(self): + for model_name in PIX2STRUCT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = Pix2StructTextModel.from_pretrained(model_name) + self.assertIsNotNone(model) + + +class Pix2StructTextImageModelsModelTester: + def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=True): + if text_kwargs is None: + text_kwargs = {} + if vision_kwargs is None: + vision_kwargs = {} + + self.parent = parent + self.text_model_tester = Pix2StructTextModelTester(parent, **text_kwargs) + self.vision_model_tester = Pix2StructVisionModelTester(parent, **vision_kwargs) + self.is_training = is_training + + def prepare_config_and_inputs(self): + text_config, input_ids, attention_mask = self.text_model_tester.prepare_config_and_inputs() + vision_config, flattened_patches = self.vision_model_tester.prepare_config_and_inputs() + + config = self.get_config(text_config, vision_config) + + return config, input_ids, attention_mask, flattened_patches + + def get_config(self, text_config, vision_config): + return Pix2StructConfig.from_text_vision_configs(text_config, vision_config, projection_dim=64) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, input_ids, decoder_attention_mask, flattened_patches = config_and_inputs + + attention_mask = (flattened_patches.sum(dim=-1) != 0).float() + + inputs_dict = { + "decoder_input_ids": input_ids, + "labels": input_ids, + "decoder_attention_mask": decoder_attention_mask, + "flattened_patches": flattened_patches, + "attention_mask": attention_mask, + } + return config, inputs_dict + + +@require_torch +class Pix2StructTextImageModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = (Pix2StructForConditionalGeneration,) if is_torch_available() else () + fx_compatible = False + test_head_masking = False + test_pruning = False + test_resize_embeddings = False + test_attention_outputs = False + test_torchscript = False + + def setUp(self): + self.model_tester = Pix2StructTextImageModelsModelTester(self) + + def test_model(self): + config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() + for model_class in self.all_model_classes: + model = model_class(config) + + output = model(**input_dict) + self.assertEqual( + output[1].shape, + ( + self.model_tester.vision_model_tester.batch_size, + self.model_tester.text_model_tester.seq_length, + self.model_tester.text_model_tester.vocab_size, + ), + ) + + @unittest.skip(reason="Hidden_states is tested in individual model tests") + def test_hidden_states_output(self): + pass + + @unittest.skip(reason="Inputs_embeds is tested in individual model tests") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="Retain_grad is tested in individual model tests") + def test_retain_grad_hidden_states_attentions(self): + pass + + @unittest.skip(reason="Pix2StructModel does not have input/output embeddings") + def test_model_common_attributes(self): + pass + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + if model.config.is_encoder_decoder: + expected_arg_names = [ + "input_ids", + "attention_mask", + "decoder_input_ids", + "decoder_attention_mask", + ] + expected_arg_names.extend( + ["head_mask", "decoder_head_mask", "cross_attn_head_mask", "encoder_outputs"] + if "head_mask" and "decoder_head_mask" and "cross_attn_head_mask" in arg_names + else ["encoder_outputs"] + ) + self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names) + else: + expected_arg_names = ( + ["input_ids"] if model_class != Pix2StructForConditionalGeneration else ["flattened_patches"] + ) + self.assertListEqual(arg_names[:1], expected_arg_names) + + def test_training(self): + if not self.model_tester.is_training: + return + + for model_class in self.all_model_classes[:-1]: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.return_dict = True + + model = model_class(config) + model.to(torch_device) + model.train() + inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + + # hardcode labels to be the same as input_ids + inputs["labels"] = inputs["input_ids"] + + loss = model(**inputs).loss + loss.backward() + + def test_training_gradient_checkpointing(self): + if not self.model_tester.is_training: + return + + for model_class in self.all_model_classes[:-1]: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.use_cache = False + config.return_dict = True + + model = model_class(config) + model.to(torch_device) + model.gradient_checkpointing_enable() + model.train() + inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + + # hardcode labels to be the same as input_ids + inputs["labels"] = inputs["input_ids"] + + loss = model(**inputs).loss + loss.backward() + + # override as the `logit_scale` parameter initilization is different for Pix2Struct + def test_initialization(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + configs_no_init = _config_zero_init(config) + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + for name, param in model.named_parameters(): + if param.requires_grad: + # check if `logit_scale` is initilized as per the original implementation + if name == "logit_scale": + self.assertAlmostEqual( + param.data.item(), + np.log(1 / 0.07), + delta=1e-3, + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + else: + self.assertIn( + ((param.data.mean() * 1e9).round() / 1e9).item(), + [0.0, 1.0], + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + + def _create_and_check_torchscript(self, config, inputs_dict): + if not self.test_torchscript: + return + + configs_no_init = _config_zero_init(config) # To be sure we have no Nan + configs_no_init.torchscript = True + configs_no_init.return_dict = False + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + model.to(torch_device) + model.eval() + + try: + input_ids = inputs_dict["input_ids"] + flattened_patches = inputs_dict["flattened_patches"] # Pix2Struct needs flattened_patches + traced_model = torch.jit.trace(model, (input_ids, flattened_patches)) + except RuntimeError: + self.fail("Couldn't trace module.") + + with tempfile.TemporaryDirectory() as tmp_dir_name: + pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt") + + try: + torch.jit.save(traced_model, pt_file_name) + except Exception: + self.fail("Couldn't save module.") + + try: + loaded_model = torch.jit.load(pt_file_name) + except Exception: + self.fail("Couldn't load module.") + + model.to(torch_device) + model.eval() + + loaded_model.to(torch_device) + loaded_model.eval() + + model_state_dict = model.state_dict() + loaded_model_state_dict = loaded_model.state_dict() + + self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys())) + + models_equal = True + for layer_name, p1 in model_state_dict.items(): + p2 = loaded_model_state_dict[layer_name] + if p1.data.ne(p2.data).sum() > 0: + models_equal = False + + self.assertTrue(models_equal) + + def test_load_vision_text_config(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # Save Pix2StructConfig and check if we can load Pix2StructVisionConfig from it + with tempfile.TemporaryDirectory() as tmp_dir_name: + config.save_pretrained(tmp_dir_name) + vision_config = Pix2StructVisionConfig.from_pretrained(tmp_dir_name) + self.assertDictEqual(config.vision_config.to_dict(), vision_config.to_dict()) + + # Save Pix2StructConfig and check if we can load Pix2StructTextConfig from it + with tempfile.TemporaryDirectory() as tmp_dir_name: + config.save_pretrained(tmp_dir_name) + text_config = Pix2StructTextConfig.from_pretrained(tmp_dir_name) + self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) + + +# We will verify our results on an image of a stop sign +def prepare_img(): + url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg" + im = Image.open(requests.get(url, stream=True).raw) + return im + + +@require_vision +@require_torch +@slow +class Pix2StructIntegrationTest(unittest.TestCase): + def test_inference_image_captioning(self): + model = Pix2StructForConditionalGeneration.from_pretrained("google/pix2struct-textcaps-base").to(torch_device) + processor = Pix2StructProcessor.from_pretrained("google/pix2struct-textcaps-base") + image = prepare_img() + + # image only + inputs = processor(images=image, return_tensors="pt").to(torch_device) + + predictions = model.generate(**inputs) + + self.assertEqual( + processor.decode(predictions[0], skip_special_tokens=True), "A stop sign is on a street corner." + ) + + def test_batched_inference_image_captioning(self): + model = Pix2StructForConditionalGeneration.from_pretrained("google/pix2struct-textcaps-base").to(torch_device) + processor = Pix2StructProcessor.from_pretrained("google/pix2struct-textcaps-base") + image_1 = prepare_img() + + second_url = ( + "https://www.connollycove.com/wp-content/uploads/2019/06/temple-bar-dublin-world-famous-irish-pub.jpg" + ) + image_2 = Image.open(requests.get(second_url, stream=True).raw) + + # image only + inputs = processor(images=[image_1, image_2], return_tensors="pt").to(torch_device) + + predictions = model.generate(**inputs) + + self.assertEqual( + processor.decode(predictions[0], skip_special_tokens=True), "A stop sign is on a street corner." + ) + + self.assertEqual( + processor.decode(predictions[1], skip_special_tokens=True), + "A row of books including The Temple Bar and Guiness.", + ) + + def test_batched_inference_image_captioning_conditioned(self): + model = Pix2StructForConditionalGeneration.from_pretrained("google/pix2struct-textcaps-base").to(torch_device) + processor = Pix2StructProcessor.from_pretrained("google/pix2struct-textcaps-base") + image_1 = prepare_img() + + second_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/temple-bar-dublin-world-famous-irish-pub.jpg" + image_2 = Image.open(requests.get(second_url, stream=True).raw) + texts = ["A picture of", "An photography of"] + + # image only + inputs = processor(images=[image_1, image_2], text=texts, return_tensors="pt").to(torch_device) + + predictions = model.generate(**inputs) + + self.assertEqual( + processor.decode(predictions[0], skip_special_tokens=True), "A picture of a stop sign that says yes." + ) + + self.assertEqual( + processor.decode(predictions[1], skip_special_tokens=True), + "An photography of the Temple Bar and a few other places.", + ) + + def test_vqa_model(self): + model_id = "ybelkada/pix2struct-ai2d-base" + + image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg" + image = Image.open(requests.get(image_url, stream=True).raw) + + model = Pix2StructForConditionalGeneration.from_pretrained(model_id, torch_dtype=torch.bfloat16).to( + torch_device + ) + processor = Pix2StructProcessor.from_pretrained(model_id) + + # image only + text = "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud" + + inputs = processor(images=image, return_tensors="pt", text=text).to(torch_device, torch.bfloat16) + + predictions = model.generate(**inputs) + self.assertEqual(processor.decode(predictions[0], skip_special_tokens=True), "ash cloud") + + def test_vqa_model_batched(self): + model_id = "ybelkada/pix2struct-ai2d-base" + + image_urls = [ + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg", + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo-2.png", + ] + + images = [Image.open(requests.get(image_url, stream=True).raw) for image_url in image_urls] + + texts = [ + "What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud", + "What is the producer in the diagram? (1) Phytoplankton (2) Zooplankton (3) Large fish (4) Small fish", + ] + + model = Pix2StructForConditionalGeneration.from_pretrained(model_id, torch_dtype=torch.bfloat16).to( + torch_device + ) + processor = Pix2StructProcessor.from_pretrained(model_id) + + inputs = processor(images=images, return_tensors="pt", text=texts).to(torch_device, torch.bfloat16) + + predictions = model.generate(**inputs) + self.assertEqual(processor.decode(predictions[0], skip_special_tokens=True), "ash cloud") + self.assertEqual(processor.decode(predictions[1], skip_special_tokens=True), "Phytoplankton") diff --git a/tests/models/pix2struct/test_processor_pix2struct.py b/tests/models/pix2struct/test_processor_pix2struct.py new file mode 100644 index 000000000000..6a7387b6ff37 --- /dev/null +++ b/tests/models/pix2struct/test_processor_pix2struct.py @@ -0,0 +1,192 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import shutil +import tempfile +import unittest + +import numpy as np +import pytest + +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import is_vision_available + + +if is_vision_available(): + from PIL import Image + + from transformers import ( + AutoProcessor, + Pix2StructImageProcessor, + Pix2StructProcessor, + PreTrainedTokenizerFast, + T5Tokenizer, + ) + + +@require_vision +@require_torch +class Pix2StructProcessorTest(unittest.TestCase): + def setUp(self): + self.tmpdirname = tempfile.mkdtemp() + + image_processor = Pix2StructImageProcessor() + tokenizer = T5Tokenizer.from_pretrained("t5-small") + + processor = Pix2StructProcessor(image_processor, tokenizer) + + processor.save_pretrained(self.tmpdirname) + + def get_tokenizer(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer + + def get_image_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor + + def tearDown(self): + shutil.rmtree(self.tmpdirname) + + def prepare_image_inputs(self): + """ + This function prepares a list of random PIL images of the same fixed size. + """ + + image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)] + + image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs] + + return image_inputs + + def test_save_load_pretrained_additional_features(self): + processor = Pix2StructProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor()) + processor.save_pretrained(self.tmpdirname) + + tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)") + image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0) + + processor = Pix2StructProcessor.from_pretrained( + self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0 + ) + + self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab()) + self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast) + + self.assertEqual(processor.image_processor.to_json_string(), image_processor_add_kwargs.to_json_string()) + self.assertIsInstance(processor.image_processor, Pix2StructImageProcessor) + + def test_image_processor(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = Pix2StructProcessor(tokenizer=tokenizer, image_processor=image_processor) + + image_input = self.prepare_image_inputs() + + input_feat_extract = image_processor(image_input, return_tensors="np") + input_processor = processor(images=image_input, return_tensors="np") + + for key in input_feat_extract.keys(): + self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2) + + def test_tokenizer(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = Pix2StructProcessor(tokenizer=tokenizer, image_processor=image_processor) + + input_str = "lower newer" + + encoded_processor = processor(text=input_str) + + encoded_tok = tokenizer(input_str, return_token_type_ids=False, add_special_tokens=False) + + for key in encoded_tok.keys(): + self.assertListEqual(encoded_tok[key], encoded_processor[key]) + + def test_processor(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = Pix2StructProcessor(tokenizer=tokenizer, image_processor=image_processor) + + input_str = "lower newer" + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input) + + self.assertListEqual( + list(inputs.keys()), ["flattened_patches", "attention_mask", "decoder_attention_mask", "decoder_input_ids"] + ) + + # test if it raises when no input is passed + with pytest.raises(ValueError): + processor() + + def test_processor_max_patches(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = Pix2StructProcessor(tokenizer=tokenizer, image_processor=image_processor) + + input_str = "lower newer" + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input) + + max_patches = [512, 1024, 2048, 4096] + expected_hidden_size = [770, 770, 770, 770] + # with text + for i, max_patch in enumerate(max_patches): + inputs = processor(text=input_str, images=image_input, max_patches=max_patch) + self.assertEqual(inputs["flattened_patches"][0].shape[0], max_patch) + self.assertEqual(inputs["flattened_patches"][0].shape[1], expected_hidden_size[i]) + + # without text input + for i, max_patch in enumerate(max_patches): + inputs = processor(images=image_input, max_patches=max_patch) + self.assertEqual(inputs["flattened_patches"][0].shape[0], max_patch) + self.assertEqual(inputs["flattened_patches"][0].shape[1], expected_hidden_size[i]) + + def test_tokenizer_decode(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = Pix2StructProcessor(tokenizer=tokenizer, image_processor=image_processor) + + predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]] + + decoded_processor = processor.batch_decode(predicted_ids) + decoded_tok = tokenizer.batch_decode(predicted_ids) + + self.assertListEqual(decoded_tok, decoded_processor) + + def test_model_input_names(self): + image_processor = self.get_image_processor() + tokenizer = self.get_tokenizer() + + processor = Pix2StructProcessor(tokenizer=tokenizer, image_processor=image_processor) + + input_str = "lower newer" + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input) + + # For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"] + self.assertListEqual( + list(inputs.keys()), ["flattened_patches", "attention_mask", "decoder_attention_mask", "decoder_input_ids"] + ) + + inputs = processor(text=input_str) + + # For now the processor supports only ["flattened_patches", "input_ids", "attention_mask", "decoder_attention_mask"] + self.assertListEqual(list(inputs.keys()), ["input_ids", "attention_mask"]) diff --git a/utils/check_repo.py b/utils/check_repo.py index 121993bc1e83..bcdecf729e4a 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -257,6 +257,9 @@ "FlaxCLIPVisionModel", "FlaxWav2Vec2ForCTC", "DetrForSegmentation", + "Pix2StructVisionModel", + "Pix2StructTextModel", + "Pix2StructForConditionalGeneration", "ConditionalDetrForSegmentation", "DPRReader", "FlaubertForQuestionAnswering", From 561f83c2442d012a32ea13bf713e8701c8924e83 Mon Sep 17 00:00:00 2001 From: Tom Aarsen <37621491+tomaarsen@users.noreply.github.com> Date: Wed, 22 Mar 2023 16:57:08 +0100 Subject: [PATCH 164/277] docs: Resolve incorrect type typo in trainer methods (#22316) Resolve incorrect type typo in trainer methods --- src/transformers/trainer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 1b3dc8055e4d..238451606188 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -2915,7 +2915,7 @@ def evaluate( Pass a dataset if you wish to override `self.eval_dataset`. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. It must implement the `__len__` method. - ignore_keys (`Lst[str]`, *optional*): + ignore_keys (`List[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. metric_key_prefix (`str`, *optional*, defaults to `"eval"`): @@ -2980,7 +2980,7 @@ def predict( test_dataset (`Dataset`): Dataset to run the predictions on. If it is an `datasets.Dataset`, columns not accepted by the `model.forward()` method are automatically removed. Has to implement the method `__len__` - ignore_keys (`Lst[str]`, *optional*): + ignore_keys (`List[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. metric_key_prefix (`str`, *optional*, defaults to `"test"`): @@ -3314,7 +3314,7 @@ def prediction_step( argument `labels`. Check your model's documentation for all accepted arguments. prediction_loss_only (`bool`): Whether or not to return the loss only. - ignore_keys (`Lst[str]`, *optional*): + ignore_keys (`List[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. From 197a6aff8488fe2ce4ed942be9804fed65ebc55c Mon Sep 17 00:00:00 2001 From: Luc CAILLIAU <74506016+luccailliau@users.noreply.github.com> Date: Wed, 22 Mar 2023 19:13:20 +0100 Subject: [PATCH 165/277] Chunkable token classification pipeline (#21771) * Chunkable classification pipeline The TokenClassificationPipeline is now able to process sequences longer than 512. No matter the framework, the model, the tokenizer. We just have to pass process_all=True and a stride number (optional). The behavior remains the same if you don't pass these optional parameters. For overlapping parts when using stride above 0, we consider only the max scores for each overlapped token in all chunks where the token is. * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * update with latest black format * update black format * Update token_classification.py * Update token_classification.py * format correction * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update comments * Update src/transformers/pipelines/token_classification.py Co-authored-by: Nicolas Patry * Update token_classification.py Correct spaces, remove process_all and keep only stride. If stride is provided, the pipeline is applied to the whole text. * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update chunk aggregation Update the chunk aggregation strategy based on entities aggregation. * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py Remove unnecessary pop from outputs dict * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update token_classification.py * Update src/transformers/pipelines/token_classification.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * add chunking tests * correct formating * correct formatting * correct model id for test chunking * update scores with nested simplify * Update test_pipelines_token_classification.py * Update test_pipelines_token_classification.py * update model to a tiny one * Update test_pipelines_token_classification.py * Adding smaller test for chunking. * Fixup * Update token_classification.py * Update src/transformers/pipelines/token_classification.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update src/transformers/pipelines/token_classification.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --------- Co-authored-by: Nicolas Patry Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- .../pipelines/token_classification.py | 156 +++++++++++++----- .../test_pipelines_token_classification.py | 121 ++++++++++++++ 2 files changed, 234 insertions(+), 43 deletions(-) diff --git a/src/transformers/pipelines/token_classification.py b/src/transformers/pipelines/token_classification.py index f3c78d3498d5..5d0244328c31 100644 --- a/src/transformers/pipelines/token_classification.py +++ b/src/transformers/pipelines/token_classification.py @@ -5,13 +5,19 @@ import numpy as np from ..models.bert.tokenization_bert import BasicTokenizer -from ..utils import ExplicitEnum, add_end_docstrings, is_tf_available, is_torch_available -from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline +from ..utils import ( + ExplicitEnum, + add_end_docstrings, + is_tf_available, + is_torch_available, +) +from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline, Dataset if is_tf_available(): - from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING + import tensorflow as tf + from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING if is_torch_available(): from ..models.auto.modeling_auto import MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING @@ -60,6 +66,9 @@ class AggregationStrategy(ExplicitEnum): grouped_entities (`bool`, *optional*, defaults to `False`): DEPRECATED, use `aggregation_strategy` instead. Whether or not to group the tokens corresponding to the same entity together in the predictions or not. + stride (`int`, *optional*): + If stride is provided, the pipeline is applied on all the text. The text is split into chunks of size + model_max_length. Works only with fast tokenizers and `aggregation_strategy` different from `NONE`. aggregation_strategy (`str`, *optional*, defaults to `"none"`): The strategy to fuse (or not) tokens based on the model prediction. @@ -82,7 +91,7 @@ class AggregationStrategy(ExplicitEnum): end up with different tags. Word entity will simply be the token with the maximum score. """, ) -class TokenClassificationPipeline(Pipeline): +class TokenClassificationPipeline(ChunkPipeline): """ Named Entity Recognition pipeline using any `ModelForTokenClassification`. See the [named entity recognition examples](../task_summary#named-entity-recognition) for more information. @@ -139,6 +148,7 @@ def _sanitize_parameters( ignore_subwords: Optional[bool] = None, aggregation_strategy: Optional[AggregationStrategy] = None, offset_mapping: Optional[List[Tuple[int, int]]] = None, + stride: Optional[int] = None, ): preprocess_params = {} if offset_mapping is not None: @@ -174,11 +184,30 @@ def _sanitize_parameters( ): raise ValueError( "Slow tokenizers cannot handle subwords. Please set the `aggregation_strategy` option" - 'to `"simple"` or use a fast tokenizer.' + ' to `"simple"` or use a fast tokenizer.' ) postprocess_params["aggregation_strategy"] = aggregation_strategy if ignore_labels is not None: postprocess_params["ignore_labels"] = ignore_labels + if stride is not None: + if aggregation_strategy == AggregationStrategy.NONE: + raise ValueError( + "`stride` was provided to process all the text but `aggregation_strategy=" + f'"{aggregation_strategy}"`, please select another one instead.' + ) + else: + if self.tokenizer.is_fast: + tokenizer_params = { + "return_overflowing_tokens": True, + "padding": True, + "stride": stride, + } + preprocess_params["tokenizer_params"] = tokenizer_params + else: + raise ValueError( + "`stride` was provided to process all the text but you're using a slow tokenizer." + " Please use a fast tokenizer." + ) return preprocess_params, {}, postprocess_params def __call__(self, inputs: Union[str, List[str]], **kwargs): @@ -213,29 +242,40 @@ def __call__(self, inputs: Union[str, List[str]], **kwargs): return super().__call__(inputs, **kwargs) - def preprocess(self, sentence, offset_mapping=None): + def preprocess(self, sentence, offset_mapping=None, **preprocess_params): + tokenizer_params = preprocess_params.pop("tokenizer_params", {}) truncation = True if self.tokenizer.model_max_length and self.tokenizer.model_max_length > 0 else False - model_inputs = self.tokenizer( + inputs = self.tokenizer( sentence, return_tensors=self.framework, truncation=truncation, return_special_tokens_mask=True, return_offsets_mapping=self.tokenizer.is_fast, + **tokenizer_params, ) - if offset_mapping: - model_inputs["offset_mapping"] = offset_mapping + inputs.pop("overflow_to_sample_mapping", None) + num_chunks = len(inputs["input_ids"]) - model_inputs["sentence"] = sentence + for i in range(num_chunks): + if self.framework == "tf": + model_inputs = {k: tf.expand_dims(v[i], 0) for k, v in inputs.items()} + else: + model_inputs = {k: v[i].unsqueeze(0) for k, v in inputs.items()} + if offset_mapping is not None: + model_inputs["offset_mapping"] = offset_mapping + model_inputs["sentence"] = sentence if i == 0 else None + model_inputs["is_last"] = i == num_chunks - 1 - return model_inputs + yield model_inputs def _forward(self, model_inputs): # Forward special_tokens_mask = model_inputs.pop("special_tokens_mask") offset_mapping = model_inputs.pop("offset_mapping", None) sentence = model_inputs.pop("sentence") + is_last = model_inputs.pop("is_last") if self.framework == "tf": - logits = self.model(model_inputs.data)[0] + logits = self.model(**model_inputs)[0] else: output = self.model(**model_inputs) logits = output["logits"] if isinstance(output, dict) else output[0] @@ -245,38 +285,67 @@ def _forward(self, model_inputs): "special_tokens_mask": special_tokens_mask, "offset_mapping": offset_mapping, "sentence": sentence, + "is_last": is_last, **model_inputs, } - def postprocess(self, model_outputs, aggregation_strategy=AggregationStrategy.NONE, ignore_labels=None): + def postprocess(self, all_outputs, aggregation_strategy=AggregationStrategy.NONE, ignore_labels=None): if ignore_labels is None: ignore_labels = ["O"] - logits = model_outputs["logits"][0].numpy() - sentence = model_outputs["sentence"] - input_ids = model_outputs["input_ids"][0] - offset_mapping = model_outputs["offset_mapping"][0] if model_outputs["offset_mapping"] is not None else None - special_tokens_mask = model_outputs["special_tokens_mask"][0].numpy() - - maxes = np.max(logits, axis=-1, keepdims=True) - shifted_exp = np.exp(logits - maxes) - scores = shifted_exp / shifted_exp.sum(axis=-1, keepdims=True) - - if self.framework == "tf": - input_ids = input_ids.numpy() - offset_mapping = offset_mapping.numpy() if offset_mapping is not None else None - - pre_entities = self.gather_pre_entities( - sentence, input_ids, scores, offset_mapping, special_tokens_mask, aggregation_strategy - ) - grouped_entities = self.aggregate(pre_entities, aggregation_strategy) - # Filter anything that is in self.ignore_labels - entities = [ - entity - for entity in grouped_entities - if entity.get("entity", None) not in ignore_labels - and entity.get("entity_group", None) not in ignore_labels - ] - return entities + all_entities = [] + for model_outputs in all_outputs: + logits = model_outputs["logits"][0].numpy() + sentence = all_outputs[0]["sentence"] + input_ids = model_outputs["input_ids"][0] + offset_mapping = ( + model_outputs["offset_mapping"][0] if model_outputs["offset_mapping"] is not None else None + ) + special_tokens_mask = model_outputs["special_tokens_mask"][0].numpy() + + maxes = np.max(logits, axis=-1, keepdims=True) + shifted_exp = np.exp(logits - maxes) + scores = shifted_exp / shifted_exp.sum(axis=-1, keepdims=True) + + if self.framework == "tf": + input_ids = input_ids.numpy() + offset_mapping = offset_mapping.numpy() if offset_mapping is not None else None + + pre_entities = self.gather_pre_entities( + sentence, input_ids, scores, offset_mapping, special_tokens_mask, aggregation_strategy + ) + grouped_entities = self.aggregate(pre_entities, aggregation_strategy) + # Filter anything that is in self.ignore_labels + entities = [ + entity + for entity in grouped_entities + if entity.get("entity", None) not in ignore_labels + and entity.get("entity_group", None) not in ignore_labels + ] + all_entities.extend(entities) + num_chunks = len(all_outputs) + if num_chunks > 1: + all_entities = self.aggregate_overlapping_entities(all_entities) + return all_entities + + def aggregate_overlapping_entities(self, entities): + if len(entities) == 0: + return entities + entities = sorted(entities, key=lambda x: x["start"]) + aggregated_entities = [] + previous_entity = entities[0] + for entity in entities: + if previous_entity["start"] <= entity["start"] < previous_entity["end"]: + current_length = entity["end"] - entity["start"] + previous_length = previous_entity["end"] - previous_entity["start"] + if current_length > previous_length: + previous_entity = entity + elif current_length == previous_length and entity["score"] > previous_entity["score"]: + previous_entity = entity + else: + aggregated_entities.append(previous_entity) + previous_entity = entity + aggregated_entities.append(previous_entity) + return aggregated_entities def gather_pre_entities( self, @@ -290,9 +359,7 @@ def gather_pre_entities( """Fuse various numpy arrays into dicts with all the information needed for aggregation""" pre_entities = [] for idx, token_scores in enumerate(scores): - # Filter special_tokens, they should only occur - # at the sentence boundaries since we're not encoding pairs of - # sentences so we don't have to keep track of those. + # Filter special_tokens if special_tokens_mask[idx]: continue @@ -317,7 +384,10 @@ def gather_pre_entities( AggregationStrategy.AVERAGE, AggregationStrategy.MAX, }: - warnings.warn("Tokenizer does not support real words, using fallback heuristic", UserWarning) + warnings.warn( + "Tokenizer does not support real words, using fallback heuristic", + UserWarning, + ) is_subword = start_ind > 0 and " " not in sentence[start_ind - 1 : start_ind + 1] if int(input_ids[idx]) == self.tokenizer.unk_token_id: diff --git a/tests/pipelines/test_pipelines_token_classification.py b/tests/pipelines/test_pipelines_token_classification.py index 39c54fddb495..fa79de2bf6e1 100644 --- a/tests/pipelines/test_pipelines_token_classification.py +++ b/tests/pipelines/test_pipelines_token_classification.py @@ -196,6 +196,127 @@ def run_aggregation_strategy(self, model, tokenizer): ) self.assertEqual(token_classifier._postprocess_params["aggregation_strategy"], AggregationStrategy.FIRST) + @slow + @require_torch + def test_chunking(self): + NER_MODEL = "elastic/distilbert-base-uncased-finetuned-conll03-english" + model = AutoModelForTokenClassification.from_pretrained(NER_MODEL) + tokenizer = AutoTokenizer.from_pretrained(NER_MODEL, use_fast=True) + tokenizer.model_max_length = 10 + stride = 5 + sentence = ( + "Hugging Face, Inc. is a French company that develops tools for building applications using machine learning. " + "The company, based in New York City was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf." + ) + + token_classifier = TokenClassificationPipeline( + model=model, tokenizer=tokenizer, aggregation_strategy="simple", stride=stride + ) + output = token_classifier(sentence) + self.assertEqual( + nested_simplify(output), + [ + {"entity_group": "ORG", "score": 0.978, "word": "hugging face, inc.", "start": 0, "end": 18}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 24, "end": 30}, + {"entity_group": "LOC", "score": 0.997, "word": "new york city", "start": 131, "end": 144}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 168, "end": 174}, + {"entity_group": "PER", "score": 0.999, "word": "clement delangue", "start": 189, "end": 205}, + {"entity_group": "PER", "score": 0.999, "word": "julien chaumond", "start": 207, "end": 222}, + {"entity_group": "PER", "score": 0.999, "word": "thomas wolf", "start": 228, "end": 239}, + ], + ) + + token_classifier = TokenClassificationPipeline( + model=model, tokenizer=tokenizer, aggregation_strategy="first", stride=stride + ) + output = token_classifier(sentence) + self.assertEqual( + nested_simplify(output), + [ + {"entity_group": "ORG", "score": 0.978, "word": "hugging face, inc.", "start": 0, "end": 18}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 24, "end": 30}, + {"entity_group": "LOC", "score": 0.997, "word": "new york city", "start": 131, "end": 144}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 168, "end": 174}, + {"entity_group": "PER", "score": 0.999, "word": "clement delangue", "start": 189, "end": 205}, + {"entity_group": "PER", "score": 0.999, "word": "julien chaumond", "start": 207, "end": 222}, + {"entity_group": "PER", "score": 0.999, "word": "thomas wolf", "start": 228, "end": 239}, + ], + ) + + token_classifier = TokenClassificationPipeline( + model=model, tokenizer=tokenizer, aggregation_strategy="max", stride=stride + ) + output = token_classifier(sentence) + self.assertEqual( + nested_simplify(output), + [ + {"entity_group": "ORG", "score": 0.978, "word": "hugging face, inc.", "start": 0, "end": 18}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 24, "end": 30}, + {"entity_group": "LOC", "score": 0.997, "word": "new york city", "start": 131, "end": 144}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 168, "end": 174}, + {"entity_group": "PER", "score": 0.999, "word": "clement delangue", "start": 189, "end": 205}, + {"entity_group": "PER", "score": 0.999, "word": "julien chaumond", "start": 207, "end": 222}, + {"entity_group": "PER", "score": 0.999, "word": "thomas wolf", "start": 228, "end": 239}, + ], + ) + + token_classifier = TokenClassificationPipeline( + model=model, tokenizer=tokenizer, aggregation_strategy="average", stride=stride + ) + output = token_classifier(sentence) + self.assertEqual( + nested_simplify(output), + [ + {"entity_group": "ORG", "score": 0.978, "word": "hugging face, inc.", "start": 0, "end": 18}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 24, "end": 30}, + {"entity_group": "LOC", "score": 0.997, "word": "new york city", "start": 131, "end": 144}, + {"entity_group": "MISC", "score": 0.999, "word": "french", "start": 168, "end": 174}, + {"entity_group": "PER", "score": 0.999, "word": "clement delangue", "start": 189, "end": 205}, + {"entity_group": "PER", "score": 0.999, "word": "julien chaumond", "start": 207, "end": 222}, + {"entity_group": "PER", "score": 0.999, "word": "thomas wolf", "start": 228, "end": 239}, + ], + ) + + @require_torch + def test_chunking_fast(self): + # Note: We cannot run the test on "conflicts" on the chunking. + # The problem is that the model is random, and thus the results do heavily + # depend on the chunking, so we cannot expect "abcd" and "bcd" to find + # the same entities. We defer to slow tests for this. + pipe = pipeline(model="hf-internal-testing/tiny-bert-for-token-classification") + sentence = "The company, based in New York City was founded in 2016 by French entrepreneurs" + + results = pipe(sentence, aggregation_strategy="first") + # This is what this random model gives on the full sentence + self.assertEqual( + nested_simplify(results), + [ + # This is 2 actual tokens + {"end": 39, "entity_group": "MISC", "score": 0.115, "start": 31, "word": "city was"}, + {"end": 79, "entity_group": "MISC", "score": 0.115, "start": 66, "word": "entrepreneurs"}, + ], + ) + + # This will force the tokenizer to split after "city was". + pipe.tokenizer.model_max_length = 12 + self.assertEqual( + pipe.tokenizer.decode(pipe.tokenizer.encode(sentence, truncation=True)), + "[CLS] the company, based in new york city was [SEP]", + ) + + stride = 4 + results = pipe(sentence, aggregation_strategy="first", stride=stride) + self.assertEqual( + nested_simplify(results), + [ + {"end": 39, "entity_group": "MISC", "score": 0.115, "start": 31, "word": "city was"}, + # This is an extra entity found by this random model, but at least both original + # entities are there + {"end": 58, "entity_group": "MISC", "score": 0.115, "start": 56, "word": "by"}, + {"end": 79, "entity_group": "MISC", "score": 0.115, "start": 66, "word": "entrepreneurs"}, + ], + ) + @require_torch @slow def test_spanish_bert(self): From b4baf0af90389d2d0c8b2a290905e226d0478204 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 22 Mar 2023 20:02:24 +0100 Subject: [PATCH 166/277] Fix PipelineTests skip conditions (#22320) * check what tests fail * Skip failing tests * Skip failing tests * Skip failing tests * Skip failing tests * clean up * clean up --------- Co-authored-by: ydshieh --- tests/models/cpm/test_tokenization_cpm.py | 6 +++++ tests/models/luke/test_modeling_luke.py | 9 +++++++ .../models/markuplm/test_modeling_markuplm.py | 8 ++++++ tests/models/mbart/test_modeling_mbart.py | 10 +++++++ tests/models/mbart/test_modeling_tf_mbart.py | 10 +++++++ tests/models/whisper/test_modeling_whisper.py | 11 ++++++++ tests/models/xlnet/test_modeling_tf_xlnet.py | 7 +++++ tests/models/xlnet/test_modeling_xlnet.py | 7 +++++ tests/test_pipeline_mixin.py | 26 +++++++++---------- 9 files changed, 81 insertions(+), 13 deletions(-) diff --git a/tests/models/cpm/test_tokenization_cpm.py b/tests/models/cpm/test_tokenization_cpm.py index 1d66778b8c7a..fa69a6aaa7dc 100644 --- a/tests/models/cpm/test_tokenization_cpm.py +++ b/tests/models/cpm/test_tokenization_cpm.py @@ -21,6 +21,12 @@ @custom_tokenizers class CpmTokenizationTest(XLNetModelTest): + # There is no `CpmModel` + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + return True + def test_pre_tokenization(self): tokenizer = CpmTokenizer.from_pretrained("TsinghuaAI/CPM-Generate") text = "Hugging Face大法好，谁用谁知道。" diff --git a/tests/models/luke/test_modeling_luke.py b/tests/models/luke/test_modeling_luke.py index 1ab23392da0b..35bdb6b6d5fa 100644 --- a/tests/models/luke/test_modeling_luke.py +++ b/tests/models/luke/test_modeling_luke.py @@ -619,6 +619,15 @@ class LukeModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_resize_embeddings = True test_head_masking = True + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name in ["QAPipelineTests", "ZeroShotClassificationPipelineTests"]: + return True + + return False + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): entity_inputs_dict = {k: v for k, v in inputs_dict.items() if k.startswith("entity")} inputs_dict = {k: v for k, v in inputs_dict.items() if not k.startswith("entity")} diff --git a/tests/models/markuplm/test_modeling_markuplm.py b/tests/models/markuplm/test_modeling_markuplm.py index 3abdb4041ad3..d52129cf2cba 100644 --- a/tests/models/markuplm/test_modeling_markuplm.py +++ b/tests/models/markuplm/test_modeling_markuplm.py @@ -299,6 +299,14 @@ class MarkupLMModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + # ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]` + # (batch of pretokenized examples). + return True + def setUp(self): self.model_tester = MarkupLMModelTester(self) self.config_tester = ConfigTester(self, config_class=MarkupLMConfig, hidden_size=37) diff --git a/tests/models/mbart/test_modeling_mbart.py b/tests/models/mbart/test_modeling_mbart.py index b52d2f04d0f0..607babb1ba88 100644 --- a/tests/models/mbart/test_modeling_mbart.py +++ b/tests/models/mbart/test_modeling_mbart.py @@ -252,6 +252,16 @@ class MBartModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixi test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name != "FeatureExtractionPipelineTests": + # IndexError: index out of range in self + return True + + return False + def setUp(self): self.model_tester = MBartModelTester(self) self.config_tester = ConfigTester(self, config_class=MBartConfig) diff --git a/tests/models/mbart/test_modeling_tf_mbart.py b/tests/models/mbart/test_modeling_tf_mbart.py index 52cd24be274b..c3e572147354 100644 --- a/tests/models/mbart/test_modeling_tf_mbart.py +++ b/tests/models/mbart/test_modeling_tf_mbart.py @@ -198,6 +198,16 @@ class TFMBartModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCas test_pruning = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name != "FeatureExtractionPipelineTests": + # Exception encountered when calling layer '...' + return True + + return False + def setUp(self): self.model_tester = TFMBartModelTester(self) self.config_tester = ConfigTester(self, config_class=MBartConfig) diff --git a/tests/models/whisper/test_modeling_whisper.py b/tests/models/whisper/test_modeling_whisper.py index 992f4564cf6e..d2784d772476 100644 --- a/tests/models/whisper/test_modeling_whisper.py +++ b/tests/models/whisper/test_modeling_whisper.py @@ -291,6 +291,17 @@ class WhisperModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMi input_name = "input_features" + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "AutomaticSpeechRecognitionPipelineTests": + # RuntimeError: The size of tensor a (1500) must match the size of tensor b (30) at non-singleton + # dimension 1 + return True + + return False + def setUp(self): self.model_tester = WhisperModelTester(self) self.config_tester = ConfigTester(self, config_class=WhisperConfig) diff --git a/tests/models/xlnet/test_modeling_tf_xlnet.py b/tests/models/xlnet/test_modeling_tf_xlnet.py index bc65b0501ecf..bbc310aa8b1c 100644 --- a/tests/models/xlnet/test_modeling_tf_xlnet.py +++ b/tests/models/xlnet/test_modeling_tf_xlnet.py @@ -363,6 +363,13 @@ class TFXLNetModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCas test_head_masking = False test_onnx = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + # Exception encountered when calling layer '...' + return True + def setUp(self): self.model_tester = TFXLNetModelTester(self) self.config_tester = ConfigTester(self, config_class=XLNetConfig, d_inner=37) diff --git a/tests/models/xlnet/test_modeling_xlnet.py b/tests/models/xlnet/test_modeling_xlnet.py index 2b99d2e17ca9..98c935cdecf1 100644 --- a/tests/models/xlnet/test_modeling_xlnet.py +++ b/tests/models/xlnet/test_modeling_xlnet.py @@ -542,6 +542,13 @@ class XLNetModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixi fx_compatible = False test_pruning = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + # IndexError: index out of range in self + return True + # XLNet has 2 QA models -> need to manually set the correct labels for one of them here def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/test_pipeline_mixin.py b/tests/test_pipeline_mixin.py index fbeac086e110..cfe10ea36a1b 100644 --- a/tests/test_pipeline_mixin.py +++ b/tests/test_pipeline_mixin.py @@ -28,7 +28,7 @@ require_torch_or_tf, require_vision, ) -from transformers.utils import direct_transformers_import +from transformers.utils import direct_transformers_import, logging from .pipelines.test_pipelines_audio_classification import AudioClassificationPipelineTests from .pipelines.test_pipelines_automatic_speech_recognition import AutomaticSpeechRecognitionPipelineTests @@ -104,6 +104,8 @@ # Dynamically import the Transformers module to grab the attribute classes of the processor form their names. transformers_module = direct_transformers_import(PATH_TO_TRANSFORMERS) +logger = logging.get_logger(__name__) + class PipelineTesterMixin: model_tester = None @@ -179,11 +181,12 @@ def run_model_pipeline_tests(self, task, repo_name, model_architecture, tokenize tokenizer_name, processor_name, ): - self.skipTest( + logger.warning( f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: test is " f"currently known to fail for: model `{model_architecture.__name__}` | tokenizer " f"`{tokenizer_name}` | processor `{processor_name}`." ) + continue self.run_pipeline_test(task, repo_name, model_architecture, tokenizer_name, processor_name) def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name, processor_name): @@ -217,26 +220,29 @@ def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name, try: processor = processor_class.from_pretrained(repo_id) except Exception: - self.skipTest( + logger.warning( f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: Could not load the " f"processor from `{repo_id}` with `{processor_name}`." ) + return # TODO: Maybe not upload such problematic tiny models to Hub. if tokenizer is None and processor is None: - self.skipTest( + logger.warning( f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: Could not find or load " f"any tokenizer / processor from `{repo_id}`." ) + return # TODO: We should check if a model file is on the Hub repo. instead. try: model = model_architecture.from_pretrained(repo_id) except Exception: - self.skipTest( + logger.warning( f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: Could not find or load " f"the model from `{repo_id}` with `{model_architecture}`." ) + return # validate validate_test_components(self, task, model, tokenizer, processor) @@ -252,10 +258,11 @@ def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name, if pipeline is None: # The test can disable itself, but it should be very marginal # Concerns: Wav2Vec2ForCTC without tokenizer test (FastTokenizer don't exist) - self.skipTest( + logger.warning( f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: Could not get the " "pipeline for testing." ) + return task_test.run_pipeline_test(pipeline, examples) @@ -429,10 +436,3 @@ def validate_test_components(test_case, task, model, tokenizer, processor): raise ValueError( "Could not determine `vocab_size` from model configuration while `tokenizer` is not `None`." ) - # TODO: Remove tiny models from the Hub which have problematic tokenizers (but still keep this block) - if config_vocab_size is not None and len(tokenizer) > config_vocab_size: - test_case.skipTest( - f"{test_case.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: tokenizer " - f"(`{tokenizer.__class__.__name__}`) has {len(tokenizer)} tokens which is greater than " - f"`config_vocab_size` ({config_vocab_size}). Something is wrong." - ) From 87c6bb5383a07ce21302c7f07e26f7b06fc71e25 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 22 Mar 2023 12:18:57 -0700 Subject: [PATCH 167/277] [deepspeed zero3] need `generate(synced_gpus=True, ...)` (#22242) * [deepspeed zero3] need generate(synced_gpus=True, ...) * fix * rework per Sylvain's suggestion * Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- docs/source/en/main_classes/deepspeed.mdx | 8 ++++++++ src/transformers/generation/utils.py | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/source/en/main_classes/deepspeed.mdx b/docs/source/en/main_classes/deepspeed.mdx index 2b5ea6a74354..2c78078d60d0 100644 --- a/docs/source/en/main_classes/deepspeed.mdx +++ b/docs/source/en/main_classes/deepspeed.mdx @@ -2268,6 +2268,14 @@ rank1: This was a very basic example and you will want to adapt it to your needs. +### `generate` nuances + +When using multiple GPUs with ZeRO Stage-3, one has to synchronize the GPUs by calling `generate(..., synced_gpus=True)`. If this is not done if one GPU finished generating before other GPUs the whole system will hang as the rest of the GPUs will not be able to received the shard of weights from the GPU that stopped generating. + +Starting from `transformers>=4.28`, if `synced_gpus` isn't explicitly specified, it'll be set to `True` automatically if these conditions are detected. But you can still override the value of `synced_gpus` if need to. + + + ## Testing Deepspeed Integration If you submit a PR that involves DeepSpeed integration please note our CircleCI PR CI setup has no GPUs, so we only run tests requiring gpus on a different CI nightly. Therefore if you get a green CI report in your PR it doesn't mean DeepSpeed tests pass. diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index eb0e0ed4c21f..6140f4cb4007 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -24,6 +24,7 @@ import torch.distributed as dist from torch import nn +from ..deepspeed import is_deepspeed_zero3_enabled from ..modeling_outputs import CausalLMOutputWithPast, Seq2SeqLMOutput from ..models.auto import ( MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING, @@ -1114,7 +1115,7 @@ def generate( logits_processor: Optional[LogitsProcessorList] = None, stopping_criteria: Optional[StoppingCriteriaList] = None, prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None, - synced_gpus: Optional[bool] = False, + synced_gpus: Optional[bool] = None, **kwargs, ) -> Union[GenerateOutput, torch.LongTensor]: r""" @@ -1160,8 +1161,11 @@ def generate( on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful for constrained generation conditioned on the prefix, as described in [Autoregressive Entity Retrieval](https://arxiv.org/abs/2010.00904). - synced_gpus (`bool`, *optional*, defaults to `False`): - Whether to continue running the while loop until max_length (needed for ZeRO stage 3) + synced_gpus (`bool`, *optional*): + Whether to continue running the while loop until max_length. Unless overridden this flag will be set to + `True` under DeepSpeed ZeRO Stage 3 multiple GPUs environment to avoid hanging if one GPU finished + generating before other GPUs. Otherwise it'll be set to `False`. + kwargs: Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder @@ -1187,6 +1191,13 @@ def generate( - [`~generation.BeamSearchEncoderDecoderOutput`], - [`~generation.BeamSampleEncoderDecoderOutput`] """ + + if synced_gpus is None: + if is_deepspeed_zero3_enabled() and dist.world_size() > 1: + synced_gpus = True + else: + synced_gpus = False + # 1. Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call self._validate_model_class() From 4e01955f2748553ea73e404ea48057693cfb788c Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 22 Mar 2023 20:45:08 -0400 Subject: [PATCH 168/277] Fix quality due to ruff release --- examples/flax/language-modeling/run_bart_dlm_flax.py | 8 +++----- examples/legacy/pytorch-lightning/run_glue.py | 2 +- examples/legacy/pytorch-lightning/run_ner.py | 4 ++-- examples/legacy/seq2seq/run_eval_search.py | 4 ++-- examples/legacy/token-classification/run_ner.py | 2 +- examples/legacy/token-classification/run_tf_ner.py | 2 +- examples/pytorch/token-classification/run_ner.py | 2 +- .../pytorch/token-classification/run_ner_no_trainer.py | 2 +- examples/research_projects/layoutlmv3/run_funsd_cord.py | 4 ++-- .../rag-end2end-retriever/finetune_rag.py | 4 ++-- examples/research_projects/rag/finetune_rag.py | 2 +- .../research_projects/seq2seq-distillation/finetune.py | 4 ++-- .../zero-shot-distillation/distill_classifier.py | 2 +- src/transformers/benchmark/benchmark_utils.py | 2 +- src/transformers/modelcard.py | 6 +++--- src/transformers/models/esm/tokenization_esm.py | 2 +- ...t_maskformer_original_pytorch_checkpoint_to_pytorch.py | 2 +- .../models/oneformer/convert_to_hf_oneformer.py | 2 +- ...speech_to_text_wav2vec2_seq2seq_original_to_pytorch.py | 2 +- src/transformers/onnx/convert.py | 6 ++---- src/transformers/pipelines/document_question_answering.py | 2 +- src/transformers/utils/generic.py | 2 +- .../mask2former/test_image_processing_mask2former.py | 4 +--- .../models/maskformer/test_image_processing_maskformer.py | 4 +--- tests/models/oneformer/test_image_processing_oneformer.py | 4 +--- tests/models/oneformer/test_processor_oneformer.py | 4 +--- .../test_pipelines_automatic_speech_recognition.py | 7 ++----- tests/pipelines/test_pipelines_image_segmentation.py | 7 ++----- 28 files changed, 40 insertions(+), 58 deletions(-) diff --git a/examples/flax/language-modeling/run_bart_dlm_flax.py b/examples/flax/language-modeling/run_bart_dlm_flax.py index 62e4e8a83936..4cb862bb37b9 100644 --- a/examples/flax/language-modeling/run_bart_dlm_flax.py +++ b/examples/flax/language-modeling/run_bart_dlm_flax.py @@ -319,15 +319,13 @@ def permute_sentences(self, input_ids): sentence_ends = np.argwhere(end_sentence_mask) sentence_ends[:, 1] += 1 example_has_multiple_sentences, num_sentences = np.unique(sentence_ends[:, 0], return_counts=True) - num_sentences_map = {sent_idx: count for sent_idx, count in zip(example_has_multiple_sentences, num_sentences)} + num_sentences_map = dict(zip(example_has_multiple_sentences, num_sentences)) num_to_permute = np.ceil(num_sentences * self.permute_sentence_ratio).astype(int) - num_to_permute_map = { - sent_idx: count for sent_idx, count in zip(example_has_multiple_sentences, num_to_permute) - } + num_to_permute_map = dict(zip(example_has_multiple_sentences, num_to_permute)) sentence_ends = np.split(sentence_ends[:, 1], np.unique(sentence_ends[:, 0], return_index=True)[1][1:]) - sentence_ends_map = {sent_idx: count for sent_idx, count in zip(example_has_multiple_sentences, sentence_ends)} + sentence_ends_map = dict(zip(example_has_multiple_sentences, sentence_ends)) for i in range(input_ids.shape[0]): if i not in example_has_multiple_sentences: diff --git a/examples/legacy/pytorch-lightning/run_glue.py b/examples/legacy/pytorch-lightning/run_glue.py index f96c5bafcdda..5f22e2fc7a13 100644 --- a/examples/legacy/pytorch-lightning/run_glue.py +++ b/examples/legacy/pytorch-lightning/run_glue.py @@ -124,7 +124,7 @@ def _eval_end(self, outputs) -> tuple: results = {**{"val_loss": val_loss_mean}, **compute_metrics(self.hparams.task, preds, out_label_ids)} - ret = {k: v for k, v in results.items()} + ret = dict(results.items()) ret["log"] = results return ret, preds_list, out_label_list diff --git a/examples/legacy/pytorch-lightning/run_ner.py b/examples/legacy/pytorch-lightning/run_ner.py index 473851edef75..7f6b00854d99 100644 --- a/examples/legacy/pytorch-lightning/run_ner.py +++ b/examples/legacy/pytorch-lightning/run_ner.py @@ -122,7 +122,7 @@ def _eval_end(self, outputs): preds = np.argmax(preds, axis=2) out_label_ids = np.concatenate([x["target"] for x in outputs], axis=0) - label_map = {i: label for i, label in enumerate(self.labels)} + label_map = dict(enumerate(self.labels)) out_label_list = [[] for _ in range(out_label_ids.shape[0])] preds_list = [[] for _ in range(out_label_ids.shape[0])] @@ -140,7 +140,7 @@ def _eval_end(self, outputs): "f1": f1_score(out_label_list, preds_list), } - ret = {k: v for k, v in results.items()} + ret = dict(results.items()) ret["log"] = results return ret, preds_list, out_label_list diff --git a/examples/legacy/seq2seq/run_eval_search.py b/examples/legacy/seq2seq/run_eval_search.py index 1ed08c2274f2..9b5debfb2795 100755 --- a/examples/legacy/seq2seq/run_eval_search.py +++ b/examples/legacy/seq2seq/run_eval_search.py @@ -34,7 +34,7 @@ def parse_search_arg(search): groups = search.split() - entries = {k: vs for k, vs in (g.split("=") for g in groups)} + entries = dict((g.split("=") for g in groups)) entry_names = list(entries.keys()) sets = [[f"--{k} {v}" for v in vs.split(":")] for k, vs in entries.items()] matrix = [list(x) for x in itertools.product(*sets)] @@ -105,7 +105,7 @@ def run_search(): col_widths = {col: len(str(col)) for col in col_names} results = [] for r in matrix: - hparams = {k: v for k, v in (x.replace("--", "").split() for x in r)} + hparams = dict((x.replace("--", "").split() for x in r)) args_exp = " ".join(r).split() args_exp.extend(["--bs", str(args.bs)]) # in case we need to reduce its size due to CUDA OOM sys.argv = args_normal + args_exp diff --git a/examples/legacy/token-classification/run_ner.py b/examples/legacy/token-classification/run_ner.py index 212ea986b424..c571d44a1203 100644 --- a/examples/legacy/token-classification/run_ner.py +++ b/examples/legacy/token-classification/run_ner.py @@ -158,7 +158,7 @@ def main(): # Prepare CONLL-2003 task labels = token_classification_task.get_labels(data_args.labels) - label_map: Dict[int, str] = {i: label for i, label in enumerate(labels)} + label_map: Dict[int, str] = dict(enumerate(labels)) num_labels = len(labels) # Load pretrained model and tokenizer diff --git a/examples/legacy/token-classification/run_tf_ner.py b/examples/legacy/token-classification/run_tf_ner.py index df4770a70fa4..a9c41d58183d 100755 --- a/examples/legacy/token-classification/run_tf_ner.py +++ b/examples/legacy/token-classification/run_tf_ner.py @@ -144,7 +144,7 @@ def main(): # Prepare Token Classification task labels = token_classification_task.get_labels(data_args.labels) - label_map: Dict[int, str] = {i: label for i, label in enumerate(labels)} + label_map: Dict[int, str] = dict(enumerate(labels)) num_labels = len(labels) # Load pretrained model and tokenizer diff --git a/examples/pytorch/token-classification/run_ner.py b/examples/pytorch/token-classification/run_ner.py index cf9b89662281..af71ade1162d 100755 --- a/examples/pytorch/token-classification/run_ner.py +++ b/examples/pytorch/token-classification/run_ner.py @@ -407,7 +407,7 @@ def get_label_list(labels): # Set the correspondences label/ID inside the model config model.config.label2id = {l: i for i, l in enumerate(label_list)} - model.config.id2label = {i: l for i, l in enumerate(label_list)} + model.config.id2label = dict(enumerate(label_list)) # Map that sends B-Xxx label to its I-Xxx counterpart b_to_i_label = [] diff --git a/examples/pytorch/token-classification/run_ner_no_trainer.py b/examples/pytorch/token-classification/run_ner_no_trainer.py index 1007ae2ca6ec..d76ee33ebc01 100755 --- a/examples/pytorch/token-classification/run_ner_no_trainer.py +++ b/examples/pytorch/token-classification/run_ner_no_trainer.py @@ -442,7 +442,7 @@ def get_label_list(labels): # Set the correspondences label/ID inside the model config model.config.label2id = {l: i for i, l in enumerate(label_list)} - model.config.id2label = {i: l for i, l in enumerate(label_list)} + model.config.id2label = dict(enumerate(label_list)) # Map that sends B-Xxx label to its I-Xxx counterpart b_to_i_label = [] diff --git a/examples/research_projects/layoutlmv3/run_funsd_cord.py b/examples/research_projects/layoutlmv3/run_funsd_cord.py index 04e4498a1a15..e826fd997424 100644 --- a/examples/research_projects/layoutlmv3/run_funsd_cord.py +++ b/examples/research_projects/layoutlmv3/run_funsd_cord.py @@ -294,11 +294,11 @@ def get_label_list(labels): if isinstance(features[label_column_name].feature, ClassLabel): label_list = features[label_column_name].feature.names # No need to convert the labels since they are already ints. - id2label = {k: v for k, v in enumerate(label_list)} + id2label = dict(enumerate(label_list)) label2id = {v: k for k, v in enumerate(label_list)} else: label_list = get_label_list(datasets["train"][label_column_name]) - id2label = {k: v for k, v in enumerate(label_list)} + id2label = dict(enumerate(label_list)) label2id = {v: k for k, v in enumerate(label_list)} num_labels = len(label_list) diff --git a/examples/research_projects/rag-end2end-retriever/finetune_rag.py b/examples/research_projects/rag-end2end-retriever/finetune_rag.py index 194eeb3fa3d7..b0a6c1831907 100644 --- a/examples/research_projects/rag-end2end-retriever/finetune_rag.py +++ b/examples/research_projects/rag-end2end-retriever/finetune_rag.py @@ -360,7 +360,7 @@ def training_step(self, batch, batch_idx) -> Dict: loss_tensors = self._step(batch) - logs = {name: loss for name, loss in zip(self.loss_names, loss_tensors)} + logs = dict(zip(self.loss_names, loss_tensors)) # tokens per batch tgt_pad_token_id = ( self.tokenizer.generator.pad_token_id @@ -434,7 +434,7 @@ def _generative_step(self, batch: dict) -> dict: target: List[str] = self.ids_to_clean_text(batch["decoder_input_ids"]) # print(preds,target) loss_tensors = self._step(batch) - base_metrics = {name: loss for name, loss in zip(self.loss_names, loss_tensors)} + base_metrics = dict(zip(self.loss_names, loss_tensors)) gen_metrics: Dict = self.calc_generative_metrics(preds, target) summ_len = np.mean(lmap(len, generated_ids)) diff --git a/examples/research_projects/rag/finetune_rag.py b/examples/research_projects/rag/finetune_rag.py index 2e058850ecf2..64116a1d53d3 100644 --- a/examples/research_projects/rag/finetune_rag.py +++ b/examples/research_projects/rag/finetune_rag.py @@ -321,7 +321,7 @@ def _generative_step(self, batch: dict) -> dict: preds: List[str] = self.ids_to_clean_text(generated_ids) target: List[str] = self.ids_to_clean_text(batch["decoder_input_ids"]) loss_tensors = self._step(batch) - base_metrics = {name: loss for name, loss in zip(self.loss_names, loss_tensors)} + base_metrics = dict(zip(self.loss_names, loss_tensors)) gen_metrics: Dict = self.calc_generative_metrics(preds, target) summ_len = np.mean(lmap(len, generated_ids)) diff --git a/examples/research_projects/seq2seq-distillation/finetune.py b/examples/research_projects/seq2seq-distillation/finetune.py index a13f9b533d56..ff889af81e36 100755 --- a/examples/research_projects/seq2seq-distillation/finetune.py +++ b/examples/research_projects/seq2seq-distillation/finetune.py @@ -170,7 +170,7 @@ def pad(self) -> int: def training_step(self, batch, batch_idx) -> Dict: loss_tensors = self._step(batch) - logs = {name: loss for name, loss in zip(self.loss_names, loss_tensors)} + logs = dict(zip(self.loss_names, loss_tensors)) # tokens per batch logs["tpb"] = batch["input_ids"].ne(self.pad).sum() + batch["labels"].ne(self.pad).sum() logs["bs"] = batch["input_ids"].shape[0] @@ -225,7 +225,7 @@ def _generative_step(self, batch: dict) -> dict: preds: List[str] = self.ids_to_clean_text(generated_ids) target: List[str] = self.ids_to_clean_text(batch["labels"]) loss_tensors = self._step(batch) - base_metrics = {name: loss for name, loss in zip(self.loss_names, loss_tensors)} + base_metrics = dict(zip(self.loss_names, loss_tensors)) rouge: Dict = self.calc_generative_metrics(preds, target) summ_len = np.mean(lmap(len, generated_ids)) base_metrics.update(gen_time=gen_time, gen_len=summ_len, preds=preds, target=target, **rouge) diff --git a/examples/research_projects/zero-shot-distillation/distill_classifier.py b/examples/research_projects/zero-shot-distillation/distill_classifier.py index 16d52214376e..3325c3aae050 100644 --- a/examples/research_projects/zero-shot-distillation/distill_classifier.py +++ b/examples/research_projects/zero-shot-distillation/distill_classifier.py @@ -303,7 +303,7 @@ def main(): student_args.student_name_or_path, num_labels=len(class_names) ) tokenizer = AutoTokenizer.from_pretrained(student_args.student_name_or_path, use_fast=data_args.use_fast_tokenizer) - model.config.id2label = {i: label for i, label in enumerate(class_names)} + model.config.id2label = dict(enumerate(class_names)) model.config.label2id = {label: i for i, label in enumerate(class_names)} # 4. train student on teacher predictions diff --git a/src/transformers/benchmark/benchmark_utils.py b/src/transformers/benchmark/benchmark_utils.py index bde10f671217..b7008a7ab755 100644 --- a/src/transformers/benchmark/benchmark_utils.py +++ b/src/transformers/benchmark/benchmark_utils.py @@ -610,7 +610,7 @@ def __init__(self, args: BenchmarkArguments = None, configs: PretrainedConfig = model_name: AutoConfig.from_pretrained(model_name) for model_name in self.args.model_names } else: - self.config_dict = {model_name: config for model_name, config in zip(self.args.model_names, configs)} + self.config_dict = dict(zip(self.args.model_names, configs)) warnings.warn( f"The class {self.__class__} is deprecated. Hugging Face Benchmarking utils" diff --git a/src/transformers/modelcard.py b/src/transformers/modelcard.py index e89216b0d8b3..5f4627c3d9b2 100644 --- a/src/transformers/modelcard.py +++ b/src/transformers/modelcard.py @@ -399,9 +399,9 @@ def create_model_index(self, metric_mapping): dataset_metadata = _listify(self.dataset_metadata) if len(dataset_args) < len(dataset_tags): dataset_args = dataset_args + [None] * (len(dataset_tags) - len(dataset_args)) - dataset_mapping = {tag: name for tag, name in zip(dataset_tags, dataset_names)} - dataset_arg_mapping = {tag: arg for tag, arg in zip(dataset_tags, dataset_args)} - dataset_metadata_mapping = {tag: metadata for tag, metadata in zip(dataset_tags, dataset_metadata)} + dataset_mapping = dict(zip(dataset_tags, dataset_names)) + dataset_arg_mapping = dict(zip(dataset_tags, dataset_args)) + dataset_metadata_mapping = dict(zip(dataset_tags, dataset_metadata)) task_mapping = { task: TASK_TAG_TO_NAME_MAPPING[task] for task in _listify(self.tasks) if task in TASK_TAG_TO_NAME_MAPPING diff --git a/src/transformers/models/esm/tokenization_esm.py b/src/transformers/models/esm/tokenization_esm.py index 232ce61fb7e0..83a1b415b00d 100644 --- a/src/transformers/models/esm/tokenization_esm.py +++ b/src/transformers/models/esm/tokenization_esm.py @@ -57,7 +57,7 @@ class EsmTokenizer(PreTrainedTokenizer): def __init__(self, vocab_file, **kwargs): super().__init__(**kwargs) self.all_tokens = load_vocab_file(vocab_file) - self._id_to_token = {ind: tok for ind, tok in enumerate(self.all_tokens)} + self._id_to_token = dict(enumerate(self.all_tokens)) self._token_to_id = {tok: ind for ind, tok in enumerate(self.all_tokens)} self.unk_token = "" self.cls_token = "" diff --git a/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py index 1942f03666c5..df7c7aa7e1d8 100644 --- a/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/maskformer/convert_maskformer_original_pytorch_checkpoint_to_pytorch.py @@ -111,7 +111,7 @@ def __call__(self, original_config: object) -> MaskFormerConfig: swin = model.SWIN dataset_catalog = MetadataCatalog.get(original_config.DATASETS.TEST[0]) - id2label = {idx: label for idx, label in enumerate(dataset_catalog.stuff_classes)} + id2label = dict(enumerate(dataset_catalog.stuff_classes)) label2id = {label: idx for idx, label in id2label.items()} config: MaskFormerConfig = MaskFormerConfig( diff --git a/src/transformers/models/oneformer/convert_to_hf_oneformer.py b/src/transformers/models/oneformer/convert_to_hf_oneformer.py index 9dbd32f9d376..cb93857ad8e4 100644 --- a/src/transformers/models/oneformer/convert_to_hf_oneformer.py +++ b/src/transformers/models/oneformer/convert_to_hf_oneformer.py @@ -122,7 +122,7 @@ def __call__(self, original_config: object, is_swin: bool) -> OneFormerConfig: model = original_config.MODEL dataset_catalog = MetadataCatalog.get(original_config.DATASETS.TEST_PANOPTIC[0]) - id2label = {idx: label for idx, label in enumerate(dataset_catalog.stuff_classes)} + id2label = dict(enumerate(dataset_catalog.stuff_classes)) label2id = {label: idx for idx, label in id2label.items()} if is_swin: diff --git a/src/transformers/models/speech_encoder_decoder/convert_speech_to_text_wav2vec2_seq2seq_original_to_pytorch.py b/src/transformers/models/speech_encoder_decoder/convert_speech_to_text_wav2vec2_seq2seq_original_to_pytorch.py index 0a4bc48dea32..5e726aa9fd90 100644 --- a/src/transformers/models/speech_encoder_decoder/convert_speech_to_text_wav2vec2_seq2seq_original_to_pytorch.py +++ b/src/transformers/models/speech_encoder_decoder/convert_speech_to_text_wav2vec2_seq2seq_original_to_pytorch.py @@ -207,7 +207,7 @@ def create_vocab_dict(dict_path): "": 3, } - vocab_dict.update({k: v for k, v in zip(words, range(4, num_words + 4))}) + vocab_dict.update(dict(zip(words, range(4, num_words + 4)))) return vocab_dict diff --git a/src/transformers/onnx/convert.py b/src/transformers/onnx/convert.py index 288c2574e190..9e9cc93c064b 100644 --- a/src/transformers/onnx/convert.py +++ b/src/transformers/onnx/convert.py @@ -179,9 +179,7 @@ def export_pytorch( f=output.as_posix(), input_names=list(config.inputs.keys()), output_names=onnx_outputs, - dynamic_axes={ - name: axes for name, axes in chain(config.inputs.items(), config.outputs.items()) - }, + dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), do_constant_folding=True, use_external_data_format=config.use_external_data_format(model.num_parameters()), enable_onnx_checker=True, @@ -208,7 +206,7 @@ def export_pytorch( f=output.as_posix(), input_names=list(config.inputs.keys()), output_names=onnx_outputs, - dynamic_axes={name: axes for name, axes in chain(config.inputs.items(), config.outputs.items())}, + dynamic_axes=dict(chain(config.inputs.items(), config.outputs.items())), do_constant_folding=True, opset_version=opset, ) diff --git a/src/transformers/pipelines/document_question_answering.py b/src/transformers/pipelines/document_question_answering.py index f8c052385c1d..78f49a5e2dad 100644 --- a/src/transformers/pipelines/document_question_answering.py +++ b/src/transformers/pipelines/document_question_answering.py @@ -418,7 +418,7 @@ def _forward(self, model_inputs): else: model_outputs = self.model(**model_inputs) - model_outputs = {k: v for (k, v) in model_outputs.items()} + model_outputs = dict(model_outputs.items()) model_outputs["p_mask"] = p_mask model_outputs["word_ids"] = word_ids model_outputs["words"] = words diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index 21e9cf514f76..3c3ade198b35 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -282,7 +282,7 @@ def update(self, *args, **kwargs): def __getitem__(self, k): if isinstance(k, str): - inner_dict = {k: v for (k, v) in self.items()} + inner_dict = dict(self.items()) return inner_dict[k] else: return self.to_tuple()[k] diff --git a/tests/models/mask2former/test_image_processing_mask2former.py b/tests/models/mask2former/test_image_processing_mask2former.py index e21c8e677094..9f76f678f32e 100644 --- a/tests/models/mask2former/test_image_processing_mask2former.py +++ b/tests/models/mask2former/test_image_processing_mask2former.py @@ -298,9 +298,7 @@ def comm_get_image_processing_inputs( high = num_labels if is_instance_map: labels_expanded = list(range(num_labels)) * 2 - instance_id_to_semantic_id = { - instance_id: label_id for instance_id, label_id in enumerate(labels_expanded) - } + instance_id_to_semantic_id = dict(enumerate(labels_expanded)) annotations = [ np.random.randint(0, high * 2, (img.size[1], img.size[0])).astype(np.uint8) for img in image_inputs ] diff --git a/tests/models/maskformer/test_image_processing_maskformer.py b/tests/models/maskformer/test_image_processing_maskformer.py index 694029603bf7..128d232d4cf8 100644 --- a/tests/models/maskformer/test_image_processing_maskformer.py +++ b/tests/models/maskformer/test_image_processing_maskformer.py @@ -298,9 +298,7 @@ def comm_get_image_processing_inputs( high = num_labels if is_instance_map: labels_expanded = list(range(num_labels)) * 2 - instance_id_to_semantic_id = { - instance_id: label_id for instance_id, label_id in enumerate(labels_expanded) - } + instance_id_to_semantic_id = dict(enumerate(labels_expanded)) annotations = [ np.random.randint(0, high * 2, (img.size[1], img.size[0])).astype(np.uint8) for img in image_inputs ] diff --git a/tests/models/oneformer/test_image_processing_oneformer.py b/tests/models/oneformer/test_image_processing_oneformer.py index 8dec5c407083..d6fc5e228b0b 100644 --- a/tests/models/oneformer/test_image_processing_oneformer.py +++ b/tests/models/oneformer/test_image_processing_oneformer.py @@ -329,9 +329,7 @@ def comm_get_image_processor_inputs( high = num_labels if is_instance_map: labels_expanded = list(range(num_labels)) * 2 - instance_id_to_semantic_id = { - instance_id: label_id for instance_id, label_id in enumerate(labels_expanded) - } + instance_id_to_semantic_id = dict(enumerate(labels_expanded)) annotations = [ np.random.randint(0, high * 2, (img.size[1], img.size[0])).astype(np.uint8) for img in image_inputs ] diff --git a/tests/models/oneformer/test_processor_oneformer.py b/tests/models/oneformer/test_processor_oneformer.py index 5ce677cba6c6..e97895d7ce01 100644 --- a/tests/models/oneformer/test_processor_oneformer.py +++ b/tests/models/oneformer/test_processor_oneformer.py @@ -401,9 +401,7 @@ def comm_get_processor_inputs(self, with_segmentation_maps=False, is_instance_ma high = num_labels if is_instance_map: labels_expanded = list(range(num_labels)) * 2 - instance_id_to_semantic_id = { - instance_id: label_id for instance_id, label_id in enumerate(labels_expanded) - } + instance_id_to_semantic_id = dict(enumerate(labels_expanded)) annotations = [ np.random.randint(0, high * 2, (img.size[1], img.size[0])).astype(np.uint8) for img in image_inputs ] diff --git a/tests/pipelines/test_pipelines_automatic_speech_recognition.py b/tests/pipelines/test_pipelines_automatic_speech_recognition.py index d266438ac3ac..6e1bbe96f648 100644 --- a/tests/pipelines/test_pipelines_automatic_speech_recognition.py +++ b/tests/pipelines/test_pipelines_automatic_speech_recognition.py @@ -56,11 +56,8 @@ @is_pipeline_test class AutomaticSpeechRecognitionPipelineTests(unittest.TestCase): - model_mapping = { - k: v - for k, v in (list(MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING.items()) if MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING else []) - + (MODEL_FOR_CTC_MAPPING.items() if MODEL_FOR_CTC_MAPPING else []) - } + model_mapping = dict((list(MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING.items()) if MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING else []) + + (MODEL_FOR_CTC_MAPPING.items() if MODEL_FOR_CTC_MAPPING else [])) def get_test_pipeline(self, model, tokenizer, processor): if tokenizer is None: diff --git a/tests/pipelines/test_pipelines_image_segmentation.py b/tests/pipelines/test_pipelines_image_segmentation.py index b06672047a31..d736b7eb4183 100644 --- a/tests/pipelines/test_pipelines_image_segmentation.py +++ b/tests/pipelines/test_pipelines_image_segmentation.py @@ -80,14 +80,11 @@ def mask_to_test_readable_only_shape(mask: Image) -> Dict: @require_timm @require_torch class ImageSegmentationPipelineTests(unittest.TestCase): - model_mapping = { - k: v - for k, v in ( + model_mapping = dict(( list(MODEL_FOR_IMAGE_SEGMENTATION_MAPPING.items()) if MODEL_FOR_IMAGE_SEGMENTATION_MAPPING else [] ) + (MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING.items() if MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING else []) - + (MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING.items() if MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING else []) - } + + (MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING.items() if MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING else [])) def get_test_pipeline(self, model, tokenizer, processor): image_segmenter = ImageSegmentationPipeline(model=model, image_processor=processor) From 9600073a705e111b6163701ad4b918269bab0609 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger Date: Wed, 22 Mar 2023 20:56:22 -0400 Subject: [PATCH 169/277] Really fix quality due to ruff release --- .../test_pipelines_automatic_speech_recognition.py | 6 ++++-- tests/pipelines/test_pipelines_image_segmentation.py | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/pipelines/test_pipelines_automatic_speech_recognition.py b/tests/pipelines/test_pipelines_automatic_speech_recognition.py index 6e1bbe96f648..5db3e3e46c72 100644 --- a/tests/pipelines/test_pipelines_automatic_speech_recognition.py +++ b/tests/pipelines/test_pipelines_automatic_speech_recognition.py @@ -56,8 +56,10 @@ @is_pipeline_test class AutomaticSpeechRecognitionPipelineTests(unittest.TestCase): - model_mapping = dict((list(MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING.items()) if MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING else []) - + (MODEL_FOR_CTC_MAPPING.items() if MODEL_FOR_CTC_MAPPING else [])) + model_mapping = dict( + (list(MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING.items()) if MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING else []) + + (MODEL_FOR_CTC_MAPPING.items() if MODEL_FOR_CTC_MAPPING else []) + ) def get_test_pipeline(self, model, tokenizer, processor): if tokenizer is None: diff --git a/tests/pipelines/test_pipelines_image_segmentation.py b/tests/pipelines/test_pipelines_image_segmentation.py index d736b7eb4183..72150fe7396b 100644 --- a/tests/pipelines/test_pipelines_image_segmentation.py +++ b/tests/pipelines/test_pipelines_image_segmentation.py @@ -80,11 +80,11 @@ def mask_to_test_readable_only_shape(mask: Image) -> Dict: @require_timm @require_torch class ImageSegmentationPipelineTests(unittest.TestCase): - model_mapping = dict(( - list(MODEL_FOR_IMAGE_SEGMENTATION_MAPPING.items()) if MODEL_FOR_IMAGE_SEGMENTATION_MAPPING else [] - ) + model_mapping = dict( + (list(MODEL_FOR_IMAGE_SEGMENTATION_MAPPING.items()) if MODEL_FOR_IMAGE_SEGMENTATION_MAPPING else []) + (MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING.items() if MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING else []) - + (MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING.items() if MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING else [])) + + (MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING.items() if MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING else []) + ) def get_test_pipeline(self, model, tokenizer, processor): image_segmenter = ImageSegmentationPipeline(model=model, image_processor=processor) From a33d4aa019c729a6c7ce78313a3c06144176b2cd Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Wed, 22 Mar 2023 18:35:04 -0700 Subject: [PATCH 170/277] [gptj] support older pytorch version (#22325) * [gptj] support older pytorch version * contributor * contributor * make copies --------- Co-authored-by: Michael Wyatt Co-authored-by: Nick Hill --- src/transformers/models/codegen/modeling_codegen.py | 2 +- src/transformers/models/gptj/modeling_gptj.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/codegen/modeling_codegen.py b/src/transformers/models/codegen/modeling_codegen.py index c095ed3e7bef..1ea9a4f3f7c7 100644 --- a/src/transformers/models/codegen/modeling_codegen.py +++ b/src/transformers/models/codegen/modeling_codegen.py @@ -55,7 +55,7 @@ def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor: inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim)) sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float() - return torch.concat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1) + return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1) # Copied from transformers.models.gptj.modeling_gptj.rotate_every_two diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index 9f2b5642efa5..6cd756bc1a33 100755 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -18,6 +18,7 @@ from typing import Optional, Tuple, Union import torch +import torch.fx import torch.utils.checkpoint from torch import nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss @@ -57,7 +58,7 @@ def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor: inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim)) sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float() - return torch.concat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1) + return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1) @torch.fx.wrap From c92d6795da4b97d5c1014c80b7ba2453ede7059d Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:34:43 +0100 Subject: [PATCH 171/277] [`MBart`] Add `accelerate` support for MBart (#22309) add `accelerate` support for MBart --- src/transformers/models/mbart/modeling_mbart.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/mbart/modeling_mbart.py b/src/transformers/models/mbart/modeling_mbart.py index 55c381a530ba..0bf3fb62f769 100755 --- a/src/transformers/models/mbart/modeling_mbart.py +++ b/src/transformers/models/mbart/modeling_mbart.py @@ -502,6 +502,7 @@ class MBartPreTrainedModel(PreTrainedModel): config_class = MBartConfig base_model_prefix = "model" supports_gradient_checkpointing = True + _no_split_modules = ["MBartDecoderLayer", "MBartAttention"] def _init_weights(self, module): std = self.config.init_std @@ -702,10 +703,10 @@ def __init__(self, config: MBartConfig, embed_tokens: Optional[nn.Embedding] = N self.max_source_positions = config.max_position_embeddings self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0 + self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim, self.padding_idx) + if embed_tokens is not None: - self.embed_tokens = embed_tokens - else: - self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim, self.padding_idx) + self.embed_tokens.weight = embed_tokens.weight self.embed_positions = MBartLearnedPositionalEmbedding( config.max_position_embeddings, @@ -793,7 +794,7 @@ def forward( embed_pos = self.embed_positions(input) - hidden_states = inputs_embeds + embed_pos + hidden_states = inputs_embeds + embed_pos.to(inputs_embeds.device) hidden_states = self.layernorm_embedding(hidden_states) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) @@ -876,10 +877,10 @@ def __init__(self, config: MBartConfig, embed_tokens: Optional[nn.Embedding] = N self.max_target_positions = config.max_position_embeddings self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 + self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, self.padding_idx) + if embed_tokens is not None: - self.embed_tokens = embed_tokens - else: - self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, self.padding_idx) + self.embed_tokens.weight = embed_tokens.weight self.embed_positions = MBartLearnedPositionalEmbedding( config.max_position_embeddings, @@ -1038,7 +1039,7 @@ def forward( # embed positions positions = self.embed_positions(input, past_key_values_length) - hidden_states = inputs_embeds + positions + hidden_states = inputs_embeds + positions.to(inputs_embeds.device) hidden_states = self.layernorm_embedding(hidden_states) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) From a53fe3dd34256e30d5f65f94652a663b9d44101a Mon Sep 17 00:00:00 2001 From: mollerup23 <69806327+mollerup23@users.noreply.github.com> Date: Thu, 23 Mar 2023 08:45:13 -0400 Subject: [PATCH 172/277] Fixed gradient checkpoint bug for TimeSeriesTransformer (#22272) * Fixed gradient checkpoint bug for this model * Updating PR indentation (maintainer feedback) * make fixup --------- Co-authored-by: younesbelkada --- .../models/informer/modeling_informer.py | 12 +++++++----- .../modeling_time_series_transformer.py | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/transformers/models/informer/modeling_informer.py b/src/transformers/models/informer/modeling_informer.py index f6422fb179ab..10524c70bc42 100644 --- a/src/transformers/models/informer/modeling_informer.py +++ b/src/transformers/models/informer/modeling_informer.py @@ -1392,6 +1392,13 @@ def forward( hidden_states = self.layernorm_embedding(hidden_states + embed_pos) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -1418,11 +1425,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): diff --git a/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py b/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py index 86071d1fb8ed..286315a19615 100644 --- a/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py +++ b/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py @@ -1118,6 +1118,13 @@ def forward( hidden_states = self.layernorm_embedding(hidden_states + embed_pos) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -1144,11 +1151,6 @@ def forward( past_key_value = past_key_values[idx] if past_key_values is not None else None if self.gradient_checkpointing and self.training: - if use_cache: - logger.warning_once( - "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." - ) - use_cache = False def create_custom_forward(module): def custom_forward(*inputs): From 4e6a9defcbb039748012b988ce39a499e02c5662 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest <42851186+lhoestq@users.noreply.github.com> Date: Thu, 23 Mar 2023 15:26:51 +0100 Subject: [PATCH 173/277] Mention why one needs to specify max_steps in Trainer (#22333) * Mention why one needs to specify max_steps in Trainer * dummy change to trigger CI --- src/transformers/trainer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 238451606188..a41d43edeb4f 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -544,7 +544,10 @@ def __init__( logger.info("max_steps is given, it will override any value given in num_train_epochs") if train_dataset is not None and not has_length(train_dataset) and args.max_steps <= 0: - raise ValueError("train_dataset does not implement __len__, max_steps has to be specified") + raise ValueError( + "The train_dataset does not implement __len__, max_steps has to be specified. " + "The number of steps needs to be known in advance for the learning rate scheduler." + ) if ( train_dataset is not None From 6744e6722b86a87283aa63d24013997e815a61e5 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Thu, 23 Mar 2023 10:34:17 -0400 Subject: [PATCH 174/277] Fix various imports (#22281) * Fix various imports * Fix copies * Fix import --- src/transformers/__init__.py | 20 ++-- src/transformers/commands/convert.py | 2 +- ..._original_pytorch_checkpoint_to_pytorch.py | 8 +- ...vert_dpr_original_checkpoint_to_pytorch.py | 2 +- src/transformers/models/mgp_str/__init__.py | 2 +- .../pipelines/automatic_speech_recognition.py | 2 +- src/transformers/utils/dummy_tf_objects.py | 104 +++++++++--------- 7 files changed, 72 insertions(+), 68 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 894ca8e40d68..1be1a6f74f9f 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -6010,16 +6010,6 @@ tf_top_k_top_p_filtering, ) from .keras_callbacks import KerasMetricCallback, PushToHubCallback - from .modeling_tf_layoutlm import ( - TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST, - TFLayoutLMForMaskedLM, - TFLayoutLMForQuestionAnswering, - TFLayoutLMForSequenceClassification, - TFLayoutLMForTokenClassification, - TFLayoutLMMainLayer, - TFLayoutLMModel, - TFLayoutLMPreTrainedModel, - ) from .modeling_tf_utils import TFPreTrainedModel, TFSequenceSummary, TFSharedEmbeddings, shape_list # TensorFlow model imports @@ -6273,6 +6263,16 @@ TFHubertModel, TFHubertPreTrainedModel, ) + from .models.layoutlm import ( + TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST, + TFLayoutLMForMaskedLM, + TFLayoutLMForQuestionAnswering, + TFLayoutLMForSequenceClassification, + TFLayoutLMForTokenClassification, + TFLayoutLMMainLayer, + TFLayoutLMModel, + TFLayoutLMPreTrainedModel, + ) from .models.layoutlmv3 import ( TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST, TFLayoutLMv3ForQuestionAnswering, diff --git a/src/transformers/commands/convert.py b/src/transformers/commands/convert.py index a8cf431cb087..b46e14f5a673 100644 --- a/src/transformers/commands/convert.py +++ b/src/transformers/commands/convert.py @@ -167,7 +167,7 @@ def run(self): convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint, self._pytorch_dump_output) elif self._model_type == "lxmert": - from ..models.lxmert.convert_lxmert_original_pytorch_checkpoint_to_pytorch import ( + from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import ( convert_lxmert_checkpoint_to_pytorch, ) diff --git a/src/transformers/models/data2vec/convert_data2vec_text_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/data2vec/convert_data2vec_text_original_pytorch_checkpoint_to_pytorch.py index 9a38b3ae0bd1..81f5cd23fb9e 100644 --- a/src/transformers/models/data2vec/convert_data2vec_text_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/data2vec/convert_data2vec_text_original_pytorch_checkpoint_to_pytorch.py @@ -24,7 +24,12 @@ from fairseq.modules import TransformerSentenceEncoderLayer from packaging import version -from transformers import Data2VecTextConfig, Data2VecTextForMaskedLM, Data2VecTextForSequenceClassification +from transformers import ( + Data2VecTextConfig, + Data2VecTextForMaskedLM, + Data2VecTextForSequenceClassification, + Data2VecTextModel, +) from transformers.models.bert.modeling_bert import ( BertIntermediate, BertLayer, @@ -35,7 +40,6 @@ # IMPORTANT: In order for this script to run, please make sure to download the dictionary: `dict.txt` from wget https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz # File copied from https://github.com/pytorch/fairseq/blob/main/examples/data2vec/models/data2vec_text.py -from transformers.models.data2vec.data2vec_text import Data2VecTextModel from transformers.utils import logging diff --git a/src/transformers/models/dpr/convert_dpr_original_checkpoint_to_pytorch.py b/src/transformers/models/dpr/convert_dpr_original_checkpoint_to_pytorch.py index 6ea85620242f..b4965857b557 100644 --- a/src/transformers/models/dpr/convert_dpr_original_checkpoint_to_pytorch.py +++ b/src/transformers/models/dpr/convert_dpr_original_checkpoint_to_pytorch.py @@ -19,7 +19,7 @@ import torch from torch.serialization import default_restore_location -from .transformers import BertConfig, DPRConfig, DPRContextEncoder, DPRQuestionEncoder, DPRReader +from transformers import BertConfig, DPRConfig, DPRContextEncoder, DPRQuestionEncoder, DPRReader CheckpointState = collections.namedtuple( diff --git a/src/transformers/models/mgp_str/__init__.py b/src/transformers/models/mgp_str/__init__.py index 01c0ab7fd9f0..1bb9ae50b291 100644 --- a/src/transformers/models/mgp_str/__init__.py +++ b/src/transformers/models/mgp_str/__init__.py @@ -41,7 +41,7 @@ if TYPE_CHECKING: from .configuration_mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig - from .processing_mgp_str.py import MgpstrProcessor + from .processing_mgp_str import MgpstrProcessor from .tokenization_mgp_str import MgpstrTokenizer try: diff --git a/src/transformers/pipelines/automatic_speech_recognition.py b/src/transformers/pipelines/automatic_speech_recognition.py index 3d1a1c73484f..b96363232fb5 100644 --- a/src/transformers/pipelines/automatic_speech_recognition.py +++ b/src/transformers/pipelines/automatic_speech_recognition.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from pyctcdecode import BeamSearchDecoderCTC - from ...feature_extraction_sequence_utils import SequenceFeatureExtractor + from ..feature_extraction_sequence_utils import SequenceFeatureExtractor logger = logging.get_logger(__name__) diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index 55eb6599f100..94d881ac75d3 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -125,58 +125,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) -TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST = None - - -class TFLayoutLMForMaskedLM(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - -class TFLayoutLMForQuestionAnswering(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - -class TFLayoutLMForSequenceClassification(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - -class TFLayoutLMForTokenClassification(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - -class TFLayoutLMMainLayer(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - -class TFLayoutLMModel(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - -class TFLayoutLMPreTrainedModel(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - class TFPreTrainedModel(metaclass=DummyObject): _backends = ["tf"] @@ -1456,6 +1404,58 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) +TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class TFLayoutLMForMaskedLM(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFLayoutLMForQuestionAnswering(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFLayoutLMForSequenceClassification(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFLayoutLMForTokenClassification(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFLayoutLMMainLayer(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFLayoutLMModel(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFLayoutLMPreTrainedModel(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST = None From 34583c133e0b1f24e8fc8d3bc37f21362ca74127 Mon Sep 17 00:00:00 2001 From: Samuel Larkin Date: Thu, 23 Mar 2023 11:14:11 -0400 Subject: [PATCH 175/277] Minor typo in pipeline FillMaskPipeline's documentation. (#22339) --- src/transformers/pipelines/fill_mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/pipelines/fill_mask.py b/src/transformers/pipelines/fill_mask.py index fd977a72f8ef..74992485d369 100644 --- a/src/transformers/pipelines/fill_mask.py +++ b/src/transformers/pipelines/fill_mask.py @@ -234,7 +234,7 @@ def __call__(self, inputs, *args, **kwargs): - **sequence** (`str`) -- The corresponding input with the mask token prediction. - **score** (`float`) -- The corresponding probability. - **token** (`int`) -- The predicted token id (to replace the masked one). - - **token** (`str`) -- The predicted token (to replace the masked one). + - **token_str** (`str`) -- The predicted token (to replace the masked one). """ outputs = super().__call__(inputs, **kwargs) if isinstance(inputs, list) and len(inputs) == 1: From e5cd78921913effcb62dbf4ab72e48e3b91af3e1 Mon Sep 17 00:00:00 2001 From: Batese2001 <69521504+Batese2001@users.noreply.github.com> Date: Thu, 23 Mar 2023 11:31:32 -0400 Subject: [PATCH 176/277] Added type hints to TFDeiTModel (#22327) * Added type hints to TFDeiTModel * make style --------- Co-authored-by: Matt --- src/transformers/models/deit/modeling_tf_deit.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/transformers/models/deit/modeling_tf_deit.py b/src/transformers/models/deit/modeling_tf_deit.py index 558de60614e0..a3d487021d4b 100644 --- a/src/transformers/models/deit/modeling_tf_deit.py +++ b/src/transformers/models/deit/modeling_tf_deit.py @@ -665,11 +665,7 @@ def call( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, training: bool = False, - ): - r""" - bool_masked_pos (`tf.Tensor` of shape `(batch_size, num_patches)`, *optional*): - Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). - """ + ) -> Union[Tuple, TFBaseModelOutputWithPooling]: outputs = self.deit( pixel_values=pixel_values, bool_masked_pos=bool_masked_pos, From 892ec86a77652834e6d950a1c009a5b49b430af3 Mon Sep 17 00:00:00 2001 From: jeffhataws <56947987+jeffhataws@users.noreply.github.com> Date: Thu, 23 Mar 2023 09:27:13 -0700 Subject: [PATCH 177/277] Fix --bf16 option support for Neuron after PR #22300 (#22307) This PR fixes the "RuntimeError: No CUDA GPUs are available" when running with --bf16 option on Neuron. Related PRs: https://github.com/huggingface/transformers/pull/20684 https://github.com/huggingface/transformers/pull/22300 --- src/transformers/trainer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index a41d43edeb4f..7267d79b3c86 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -588,7 +588,12 @@ def __init__( if args.fp16 or args.bf16: if args.half_precision_backend == "auto": - if args.device == torch.device("cpu"): + if is_torch_neuroncore_available(): + if args.fp16: + raise ValueError("Tried to use `fp16` but this option is not yet supported on Neuron.") + else: + args.half_precision_backend = "cpu_amp" + elif args.device == torch.device("cpu"): if args.fp16: raise ValueError("Tried to use `fp16` but it is not supported on cpu") elif _is_native_cpu_amp_available: From 505628fbc52c78b87366668beb86d3a6fee3f7f9 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Thu, 23 Mar 2023 17:00:22 +0000 Subject: [PATCH 178/277] Generate: add test for left-padding support (#22322) --- tests/generation/test_utils.py | 48 +++++++++++++++++++ .../models/gpt_neox/test_modeling_gpt_neox.py | 3 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index a269b68d0020..c0278f6ae467 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -1497,6 +1497,54 @@ def test_generate_with_head_masking(self): attn_weights = out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1] self.assertEqual(sum([w.sum().item() for w in attn_weights]), 0.0) + # TODO (joao): this test is actually not slow :) However, it is not passing in some models (e.g. GPTNeoX) and the + # fix for some models is quite lengthy. Being slow means it doesn't block our push CI while we fix it. + @slow + def test_left_padding_compatibility(self): + # The check done in this test is fairly difficult -- depending on the model architecture, passing the right + # position index for the position embeddings can still result in a different output, due to numerical masking. + # On the other hand, for some types of position embeddings, an incorrect position index can have a minimal + # impact on the output. + # There are two tricks employed to check whether left-padding compatibility is in place: + # 1 - To reduce the negative impact of the numerical attention mask on a correct position index, we set the + # padding size to 1. + # 2 - To reduce the chance of false positives (i.e. passing when it should be failing), we run the check + # multiple times with random inputs, and it has to pass with all of them. + # NOTE: because of 2), there is some chance of false positives in this test. + + for model_class in self.all_generative_model_classes: + config, _, _, _ = self._get_input_ids_and_config() + if config.is_encoder_decoder: + continue # skip for encoder-decoder models -- they don't need left-padding compatibility + model = model_class(config).to(torch_device).eval() + signature = inspect.signature(model.forward).parameters.keys() + + no_failures = True + for _ in range(10): # there may be false positives with 10 runs, we rely on the CI to catch the flakiness + _, input_ids, attention_mask, _ = self._get_input_ids_and_config() + model_kwargs = {"input_ids": input_ids, "attention_mask": attention_mask} + if "position_ids" in signature: + position_ids = torch.cumsum(attention_mask, dim=-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + model_kwargs["position_ids"] = position_ids + next_logits_wo_padding = model(**model_kwargs).logits[:, -1, :] + + pad_size = (input_ids.shape[0], 1) + padding = torch.ones(pad_size, dtype=input_ids.dtype, device=torch_device) * config.pad_token_id + padded_input_ids = torch.cat((padding, input_ids), dim=1) + padded_attention_mask = torch.cat((torch.zeros_like(padding), attention_mask), dim=1) + model_kwargs = {"input_ids": padded_input_ids, "attention_mask": padded_attention_mask} + if "position_ids" in signature: + position_ids = torch.cumsum(padded_attention_mask, dim=-1) - 1 + position_ids.masked_fill_(padded_attention_mask == 0, 1) + model_kwargs["position_ids"] = position_ids + next_logits_with_padding = model(**model_kwargs).logits[:, -1, :] + if not torch.allclose(next_logits_wo_padding, next_logits_with_padding): + no_failures = False + break + + self.assertTrue(no_failures) + def _check_outputs(self, output, input_ids, config, use_cache=False, num_return_sequences=1): batch_size, seq_length = input_ids.shape num_sequences_in_output = batch_size * num_return_sequences diff --git a/tests/models/gpt_neox/test_modeling_gpt_neox.py b/tests/models/gpt_neox/test_modeling_gpt_neox.py index 4698b6525ccc..89765e05610b 100644 --- a/tests/models/gpt_neox/test_modeling_gpt_neox.py +++ b/tests/models/gpt_neox/test_modeling_gpt_neox.py @@ -20,6 +20,7 @@ from transformers import GPTNeoXConfig, is_torch_available from transformers.testing_utils import require_torch, torch_device +from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask from ...test_pipeline_mixin import PipelineTesterMixin @@ -186,7 +187,7 @@ def prepare_config_and_inputs_for_common(self): @require_torch -class GPTNeoXModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): +class GPTNeoXModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): all_model_classes = (GPTNeoXModel, GPTNeoXForCausalLM) if is_torch_available() else () all_generative_model_classes = (GPTNeoXForCausalLM,) if is_torch_available() else () pipeline_model_mapping = ( From 8c55fe796f4507c7cd393d9a088249781e1f6dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BA=D1=B3=D1=B3s=D0=BD=C4=AB?= <1934337+kooshi@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:15:51 -0500 Subject: [PATCH 179/277] Enable training Llama with model or pipeline parallelism (#22329) * Llama - Move target tokens to final pipeline device if needed * Update src/transformers/models/llama/modeling_llama.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update src/transformers/models/llama/modeling_llama.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/models/llama/modeling_llama.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index d46ae1ee09f3..f069a33f8e64 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -783,7 +783,11 @@ def forward( shift_labels = labels[..., 1:].contiguous() # Flatten the tokens loss_fct = CrossEntropyLoss() - loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1)) + shift_logits = shift_logits.view(-1, self.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model/pipeline parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) if not return_dict: output = (logits,) + outputs[1:] From 63d4c0eda61b73c440b9e2bbd5c6d511cff9d32d Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 23 Mar 2023 19:14:17 +0100 Subject: [PATCH 180/277] Automatically create/update tiny models (#22275) * Automatically create or update tiny models * Skip failed tests * update workflow file * use revision --------- Co-authored-by: ydshieh --- .github/workflows/update_tiny_models.yml | 47 + tests/models/altclip/test_modeling_altclip.py | 9 + ..._modeling_audio_spectrogram_transformer.py | 9 + .../test_modeling_blenderbot_small.py | 9 + tests/models/deta/test_modeling_deta.py | 9 + tests/models/ernie_m/test_modeling_ernie_m.py | 9 + .../oneformer/test_modeling_oneformer.py | 9 + .../models/splinter/test_modeling_splinter.py | 9 + tests/test_pipeline_mixin.py | 18 +- tests/utils/tiny_model_summary.json | 5926 ++++++++++++----- utils/create_dummy_models.py | 275 +- utils/update_tiny_models.py | 219 + 12 files changed, 4951 insertions(+), 1597 deletions(-) create mode 100644 .github/workflows/update_tiny_models.yml create mode 100644 utils/update_tiny_models.py diff --git a/.github/workflows/update_tiny_models.yml b/.github/workflows/update_tiny_models.yml new file mode 100644 index 000000000000..c02196077508 --- /dev/null +++ b/.github/workflows/update_tiny_models.yml @@ -0,0 +1,47 @@ +name: Self-hosted runner (push) + +on: + push: + branches: + - update_tiny_models* + repository_dispatch: + schedule: + - cron: "0 2 * * *" + +env: + TOKEN: ${{ secrets.SYLVAIN_HF_TOKEN }} + +jobs: + update_tiny_models: + name: Update tiny models + runs-on: ubuntu-latest + steps: + - name: Checkout transformers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Install + run: | + python -m pip install -U .[dev] + python -m pip install -U natten + + - name: Update tiny models + run: | + python utils/update_tiny_models.py + + - name: Full report + run: cat tiny_models/reports/tiny_model_creation_report.json + + - name: Failure report + run: cat tiny_models/reports/simple_failed_report.txt + + - name: Summary report + run: cat tiny_models/reports/tiny_model_summary.json + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: tiny_model_creation_reports + path: tiny_models/reports diff --git a/tests/models/altclip/test_modeling_altclip.py b/tests/models/altclip/test_modeling_altclip.py index a032ec1cee16..28213de84df6 100755 --- a/tests/models/altclip/test_modeling_altclip.py +++ b/tests/models/altclip/test_modeling_altclip.py @@ -402,6 +402,15 @@ class AltCLIPModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) test_resize_embeddings = False test_attention_outputs = False + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FeatureExtractionPipelineTests": + return True + + return False + def setUp(self): self.model_tester = AltCLIPModelTester(self) diff --git a/tests/models/audio_spectrogram_transformer/test_modeling_audio_spectrogram_transformer.py b/tests/models/audio_spectrogram_transformer/test_modeling_audio_spectrogram_transformer.py index 9b7238156ef2..c36e946f19f7 100644 --- a/tests/models/audio_spectrogram_transformer/test_modeling_audio_spectrogram_transformer.py +++ b/tests/models/audio_spectrogram_transformer/test_modeling_audio_spectrogram_transformer.py @@ -165,6 +165,15 @@ class ASTModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): test_resize_embeddings = False test_head_masking = False + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "AudioClassificationPipelineTests": + return True + + return False + def setUp(self): self.model_tester = ASTModelTester(self) self.config_tester = ConfigTester(self, config_class=ASTConfig, has_text_modality=False, hidden_size=37) diff --git a/tests/models/blenderbot_small/test_modeling_blenderbot_small.py b/tests/models/blenderbot_small/test_modeling_blenderbot_small.py index 71481b34281c..d2cfa94c190b 100644 --- a/tests/models/blenderbot_small/test_modeling_blenderbot_small.py +++ b/tests/models/blenderbot_small/test_modeling_blenderbot_small.py @@ -237,6 +237,15 @@ class BlenderbotSmallModelTest(ModelTesterMixin, GenerationTesterMixin, Pipeline test_pruning = False test_missing_keys = False + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "TextGenerationPipelineTests": + return True + + return False + def setUp(self): self.model_tester = BlenderbotSmallModelTester(self) self.config_tester = ConfigTester(self, config_class=BlenderbotSmallConfig) diff --git a/tests/models/deta/test_modeling_deta.py b/tests/models/deta/test_modeling_deta.py index 87656b0988d4..87cbd950c853 100644 --- a/tests/models/deta/test_modeling_deta.py +++ b/tests/models/deta/test_modeling_deta.py @@ -183,6 +183,15 @@ class DetaModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin test_head_masking = False test_missing_keys = False + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "ObjectDetectionPipelineTests": + return True + + return False + # special case for head models def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) diff --git a/tests/models/ernie_m/test_modeling_ernie_m.py b/tests/models/ernie_m/test_modeling_ernie_m.py index d86083d7f185..5e0ac95233d9 100644 --- a/tests/models/ernie_m/test_modeling_ernie_m.py +++ b/tests/models/ernie_m/test_modeling_ernie_m.py @@ -250,6 +250,15 @@ class ErnieMModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): ) test_torchscript = False + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "QAPipelineTests": + return True + + return False + def setUp(self): self.model_tester = ErnieMModelTester(self) self.config_tester = ConfigTester(self, config_class=ErnieMConfig, hidden_size=37) diff --git a/tests/models/oneformer/test_modeling_oneformer.py b/tests/models/oneformer/test_modeling_oneformer.py index f16c165380f1..dc25a2b48ff3 100644 --- a/tests/models/oneformer/test_modeling_oneformer.py +++ b/tests/models/oneformer/test_modeling_oneformer.py @@ -231,6 +231,15 @@ class OneFormerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCas test_head_masking = False test_missing_keys = False + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "FeatureExtractionPipelineTests": + return True + + return False + def setUp(self): self.model_tester = OneFormerModelTester(self) self.config_tester = ConfigTester(self, config_class=OneFormerConfig, has_text_modality=False) diff --git a/tests/models/splinter/test_modeling_splinter.py b/tests/models/splinter/test_modeling_splinter.py index c44fed33ac93..bf234b9e61e6 100644 --- a/tests/models/splinter/test_modeling_splinter.py +++ b/tests/models/splinter/test_modeling_splinter.py @@ -224,6 +224,15 @@ class SplinterModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase else {} ) + # TODO: Fix the failed tests when this model gets more usage + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "QAPipelineTests": + return True + + return False + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): inputs_dict = copy.deepcopy(inputs_dict) if return_labels: diff --git a/tests/test_pipeline_mixin.py b/tests/test_pipeline_mixin.py index cfe10ea36a1b..c047b959d5ef 100644 --- a/tests/test_pipeline_mixin.py +++ b/tests/test_pipeline_mixin.py @@ -142,18 +142,22 @@ def run_task_tests(self, task): tokenizer_names = [] processor_names = [] + commit = None if model_arch_name in tiny_model_summary: tokenizer_names = tiny_model_summary[model_arch_name]["tokenizer_classes"] processor_names = tiny_model_summary[model_arch_name]["processor_classes"] + commit = tiny_model_summary[model_arch_name]["sha"] # Adding `None` (if empty) so we can generate tests tokenizer_names = [None] if len(tokenizer_names) == 0 else tokenizer_names processor_names = [None] if len(processor_names) == 0 else processor_names repo_name = f"tiny-random-{model_arch_name}" - self.run_model_pipeline_tests(task, repo_name, model_architecture, tokenizer_names, processor_names) + self.run_model_pipeline_tests( + task, repo_name, model_architecture, tokenizer_names, processor_names, commit + ) - def run_model_pipeline_tests(self, task, repo_name, model_architecture, tokenizer_names, processor_names): + def run_model_pipeline_tests(self, task, repo_name, model_architecture, tokenizer_names, processor_names, commit): """Run pipeline tests for a specific `task` with the give model class and tokenizer/processor class names Args: @@ -187,9 +191,9 @@ def run_model_pipeline_tests(self, task, repo_name, model_architecture, tokenize f"`{tokenizer_name}` | processor `{processor_name}`." ) continue - self.run_pipeline_test(task, repo_name, model_architecture, tokenizer_name, processor_name) + self.run_pipeline_test(task, repo_name, model_architecture, tokenizer_name, processor_name, commit) - def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name, processor_name): + def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name, processor_name, commit): """Run pipeline tests for a specific `task` with the give model class and tokenizer/processor class name The model will be loaded from a model repository on the Hub. @@ -211,14 +215,14 @@ def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name, tokenizer = None if tokenizer_name is not None: tokenizer_class = getattr(transformers_module, tokenizer_name) - tokenizer = tokenizer_class.from_pretrained(repo_id) + tokenizer = tokenizer_class.from_pretrained(repo_id, revision=commit) processor = None if processor_name is not None: processor_class = getattr(transformers_module, processor_name) # If the required packages (like `Pillow` or `torchaudio`) are not installed, this will fail. try: - processor = processor_class.from_pretrained(repo_id) + processor = processor_class.from_pretrained(repo_id, revision=commit) except Exception: logger.warning( f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: Could not load the " @@ -236,7 +240,7 @@ def run_pipeline_test(self, task, repo_name, model_architecture, tokenizer_name, # TODO: We should check if a model file is on the Hub repo. instead. try: - model = model_architecture.from_pretrained(repo_id) + model = model_architecture.from_pretrained(repo_id, revision=commit) except Exception: logger.warning( f"{self.__class__.__name__}::test_pipeline_{task.replace('-', '_')} is skipped: Could not find or load " diff --git a/tests/utils/tiny_model_summary.json b/tests/utils/tiny_model_summary.json index d2d2ff2146f6..952b599db5f7 100644 --- a/tests/utils/tiny_model_summary.json +++ b/tests/utils/tiny_model_summary.json @@ -1,1749 +1,2409 @@ { - "AlbertModel": { - "tokenizer_classes": [ - "AlbertTokenizerFast" + "ASTForAudioClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "ASTFeatureExtractor" ], - "processor_classes": [] - }, - "BartModel": { - "tokenizer_classes": [ - "BartTokenizerFast", - "BartTokenizer" + "model_classes": [ + "ASTForAudioClassification" ], - "processor_classes": [] + "sha": "83d6e076db7768a3645401bad3204624985e1d08" }, - "BeitModel": { + "ASTModel": { "tokenizer_classes": [], "processor_classes": [ - "BeitImageProcessor" - ] - }, - "BertLMHeadModel": { - "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "ASTFeatureExtractor" + ], + "model_classes": [ + "ASTModel" ], - "processor_classes": [] + "sha": "75e68f956f6f2c0709b01e596e7a6aecb1b29dce" }, - "BertModel": { + "AlbertForMaskedLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "AlbertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "AlbertForMaskedLM", + "TFAlbertForMaskedLM" ], - "processor_classes": [] + "sha": "75ab12f94d4a1edd9610636547c5fb515e240e2b" }, - "BigBirdModel": { + "AlbertForMultipleChoice": { "tokenizer_classes": [ - "BigBirdTokenizerFast" + "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "AlbertForMultipleChoice", + "TFAlbertForMultipleChoice" + ], + "sha": "ba1531e4373cccce03195928b3ba2f6825311980" }, - "BigBirdPegasusModel": { + "AlbertForPreTraining": { "tokenizer_classes": [ - "PegasusTokenizerFast" + "AlbertTokenizerFast" ], - "processor_classes": [] - }, - "BlenderbotSmallModel": { - "tokenizer_classes": [], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "AlbertForPreTraining", + "TFAlbertForPreTraining" + ], + "sha": "6022449842a83d9cea298c4fbaf1e1e1c0db3568" }, - "BlenderbotModel": { + "AlbertForQuestionAnswering": { "tokenizer_classes": [ - "BlenderbotTokenizerFast", - "BlenderbotTokenizer" + "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "AlbertForQuestionAnswering", + "TFAlbertForQuestionAnswering" + ], + "sha": "1b6584d6a267dae8ff20b9f89e2b424a7972fb45" }, - "BlipModel": { + "AlbertForSequenceClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "AlbertTokenizerFast" ], - "processor_classes": [ - "BlipImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "AlbertForSequenceClassification", + "TFAlbertForSequenceClassification" + ], + "sha": "1e709531344ee0e4a34777c79507a07b69130958" }, - "BlipForConditionalGeneration": { + "AlbertForTokenClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "AlbertTokenizerFast" ], - "processor_classes": [ - "BlipImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "AlbertForTokenClassification", + "TFAlbertForTokenClassification" + ], + "sha": "f6c0d721d6d9f0751ab975148932948d5853fcc8" }, - "Blip2Model": { + "AlbertModel": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "AlbertTokenizerFast" ], - "processor_classes": [ - "BlipImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "AlbertModel", + "TFAlbertModel" + ], + "sha": "62974edf8b7246a943f6ecc8a3f7bfca052351ff" }, - "Blip2ForConditionalGeneration": { + "AlignModel": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "BertTokenizer", + "BertTokenizerFast" ], "processor_classes": [ - "BlipImageProcessor" - ] - }, - "BloomModel": { - "tokenizer_classes": [ - "BloomTokenizerFast" + "EfficientNetImageProcessor" ], - "processor_classes": [] - }, - "CanineModel": { - "tokenizer_classes": [ - "CanineTokenizer" + "model_classes": [ + "AlignModel" ], - "processor_classes": [] + "sha": "68a4f9d3f493f44efa7c1dde6fcca23350e2c92b" }, - "CLIPModel": { + "AltCLIPModel": { "tokenizer_classes": [ - "CLIPTokenizerFast", - "CLIPTokenizer" + "XLMRobertaTokenizerFast" ], "processor_classes": [ "CLIPImageProcessor" - ] + ], + "model_classes": [ + "AltCLIPModel" + ], + "sha": "3106af0fd503970717c05f27218e5cacf19ba872" }, - "CodeGenModel": { + "BartForCausalLM": { "tokenizer_classes": [ - "CodeGenTokenizerFast", - "CodeGenTokenizer" + "BartTokenizer", + "BartTokenizerFast" ], - "processor_classes": [] - }, - "ConditionalDetrModel": { - "tokenizer_classes": [], - "processor_classes": [ - "ConditionalDetrFeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "BartForCausalLM" + ], + "sha": "6ca393c5c34d638e70bafdc02488b65b9025872c" }, - "ConvBertModel": { + "BartForConditionalGeneration": { "tokenizer_classes": [ - "ConvBertTokenizerFast", - "ConvBertTokenizer" + "BartTokenizer", + "BartTokenizerFast" ], - "processor_classes": [] - }, - "ConvNextModel": { - "tokenizer_classes": [], - "processor_classes": [ - "ConvNextImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BartForConditionalGeneration", + "TFBartForConditionalGeneration" + ], + "sha": "44a5e3a5616b22b89cb767ac8d05f360e5de2e58" }, - "CTRLLMHeadModel": { + "BartForQuestionAnswering": { "tokenizer_classes": [ - "CTRLTokenizer" + "BartTokenizer", + "BartTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BartForQuestionAnswering" ], - "processor_classes": [] + "sha": "291888e031ae29b9defb5a4376460107cfb7a1a9" }, - "CTRLModel": { + "BartForSequenceClassification": { "tokenizer_classes": [ - "CTRLTokenizer" + "BartTokenizer", + "BartTokenizerFast" ], - "processor_classes": [] - }, - "CvtModel": { - "tokenizer_classes": [], - "processor_classes": [ - "ConvNextImageProcessor" - ] - }, - "Data2VecAudioModel": { - "tokenizer_classes": [], - "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "BartForSequenceClassification" + ], + "sha": "5ceca1f5dbcf32c04ef44355e4bc66128cd4ea8b" }, - "Data2VecTextModel": { + "BartModel": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "BartTokenizer", + "BartTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BartModel", + "TFBartModel" + ], + "sha": "26c409f22daa4773a78d7a7c80510cdc8b752a3d" }, - "Data2VecVisionModel": { + "BeitForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "BeitImageProcessor" - ] - }, - "DebertaV2Model": { - "tokenizer_classes": [ - "DebertaV2TokenizerFast" ], - "processor_classes": [] - }, - "DebertaModel": { - "tokenizer_classes": [ - "DebertaTokenizerFast", - "DebertaTokenizer" + "model_classes": [ + "BeitForImageClassification" ], - "processor_classes": [] + "sha": "e997587bb890f82faad4bd25eb23d85ba21ecaaa" }, - "DeformableDetrModel": { - "tokenizer_classes": [], - "processor_classes": [ - "DeformableDetrFeatureExtractor" - ] - }, - "DeiTModel": { + "BeitForSemanticSegmentation": { "tokenizer_classes": [], "processor_classes": [ - "DeiTImageProcessor" - ] + "BeitImageProcessor" + ], + "model_classes": [ + "BeitForSemanticSegmentation" + ], + "sha": "d4afa9e21e3fe5b087578ed68974d9b3ffc1fb22" }, - "DetrModel": { + "BeitModel": { "tokenizer_classes": [], "processor_classes": [ - "DetrFeatureExtractor" - ] + "BeitImageProcessor" + ], + "model_classes": [ + "BeitModel" + ], + "sha": "5c4a051f0cca6f64d02c6168deb88413cae10d2c" }, - "DistilBertModel": { + "BertForMaskedLM": { "tokenizer_classes": [ - "DistilBertTokenizerFast", - "DistilBertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] - }, - "DonutSwinModel": { - "tokenizer_classes": [], - "processor_classes": [ - "DonutFeatureExtractor" - ] - }, - "DPTModel": { - "tokenizer_classes": [], - "processor_classes": [ - "DPTImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BertForMaskedLM", + "TFBertForMaskedLM" + ], + "sha": "3e32baa52ce044c75edfb5c28abd51ee8d051282" }, - "ElectraModel": { + "BertForMultipleChoice": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BertForMultipleChoice", + "TFBertForMultipleChoice" ], - "processor_classes": [] + "sha": "0b8c3a6d411d1e19e5fd98d4d8631ae7616eeeaa" }, - "ErnieModel": { + "BertForNextSentencePrediction": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BertForNextSentencePrediction", + "TFBertForNextSentencePrediction" + ], + "sha": "628e70debf8864bd0b63aff7901d17d9c4f7612c" }, - "EsmModel": { + "BertForPreTraining": { "tokenizer_classes": [ - "EsmTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BertForPreTraining", + "TFBertForPreTraining" + ], + "sha": "c748ad37e6a200a6f64b2764191bfe13f976032f" }, - "FlaubertModel": { + "BertForQuestionAnswering": { "tokenizer_classes": [ - "FlaubertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BertForQuestionAnswering", + "TFBertForQuestionAnswering" + ], + "sha": "4671ad0c21493b97c5eb2f0201192704c29876d5" }, - "FlaubertWithLMHeadModel": { + "BertForSequenceClassification": { "tokenizer_classes": [ - "FlaubertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BertForSequenceClassification", + "TFBertForSequenceClassification" + ], + "sha": "37a9d44022264c12bdf3ec257778f953b63d4aaf" }, - "FlavaModel": { + "BertForTokenClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [ - "FlavaImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BertForTokenClassification", + "TFBertForTokenClassification" + ], + "sha": "d7dc3a0793ff6dfcb794b21130ee0f185d2c61a2" }, - "FNetModel": { + "BertLMHeadModel": { "tokenizer_classes": [ - "FNetTokenizerFast" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BertLMHeadModel", + "TFBertLMHeadModel" ], - "processor_classes": [] + "sha": "b4e3acc1990f3e365ffddbd54b620a26d9fb4b09" }, - "FSMTModel": { + "BertModel": { "tokenizer_classes": [ - "FSMTTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BertModel", + "TFBertModel" ], - "processor_classes": [] + "sha": "3956d303d3cddf0708ff20660c1ea5f6ec30e434" }, - "FunnelBaseModel": { + "BigBirdForCausalLM": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "BigBirdTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BigBirdForCausalLM" ], - "processor_classes": [] + "sha": "04578a05f11c0006e4e5deaf38b48889a8dc8f4f" }, - "FunnelModel": { + "BigBirdForMaskedLM": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "BigBirdTokenizerFast" ], - "processor_classes": [] - }, - "GLPNModel": { - "tokenizer_classes": [], - "processor_classes": [ - "GLPNImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BigBirdForMaskedLM" + ], + "sha": "52b8e93488b5d8235711543d4671ea08ea3f9560" }, - "GPT2LMHeadModel": { + "BigBirdForMultipleChoice": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "BigBirdTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BigBirdForMultipleChoice" + ], + "sha": "56459f9bcde6a36870e4d743295f6ce69ba5fc7b" }, - "GPT2Model": { + "BigBirdForPreTraining": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "BigBirdTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BigBirdForPreTraining" + ], + "sha": "49f55d7252dd9151722b330fa02073c6d809c55e" }, - "GPTNeoModel": { + "BigBirdForQuestionAnswering": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "BigBirdTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BigBirdForQuestionAnswering" + ], + "sha": "a5b3c8567610d4dde63c282d6fb6fd2ec04cbb39" }, - "GPTNeoXModel": { + "BigBirdForSequenceClassification": { "tokenizer_classes": [ - "GPTNeoXTokenizerFast" + "BigBirdTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BigBirdForSequenceClassification" ], - "processor_classes": [] + "sha": "2b29389f623fa7af3ffb08c51e9bcbda270ae9ee" }, - "GPTNeoXJapaneseModel": { + "BigBirdForTokenClassification": { "tokenizer_classes": [ - "GPTNeoXJapaneseTokenizer" + "BigBirdTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BigBirdForTokenClassification" + ], + "sha": "beda63d67d07b133e603f51aea6b84cae29b9ea7" }, - "GPTJModel": { + "BigBirdModel": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "BigBirdTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BigBirdModel" + ], + "sha": "814adde7ccd69821684a0c6124401f0c180d700c" }, - "GroupViTModel": { + "BigBirdPegasusForCausalLM": { "tokenizer_classes": [ - "CLIPTokenizerFast", - "CLIPTokenizer" + "PegasusTokenizerFast" ], - "processor_classes": [ - "CLIPImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BigBirdPegasusForCausalLM" + ], + "sha": "e1a5b87220073127f718fec558cbc86795b6ed61" }, - "HubertModel": { + "BigBirdPegasusForConditionalGeneration": { "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" + "PegasusTokenizerFast" ], - "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "BigBirdPegasusForConditionalGeneration" + ], + "sha": "40fad528589229426174241a641034f1f971a2b7" }, - "IBertModel": { + "BigBirdPegasusForQuestionAnswering": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "PegasusTokenizerFast" ], - "processor_classes": [] - }, - "ImageGPTModel": { - "tokenizer_classes": [], - "processor_classes": [ - "ImageGPTImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BigBirdPegasusForQuestionAnswering" + ], + "sha": "18d836e06a02bd1dc36af9a5eeaf3d326b1d368a" }, - "LayoutLMModel": { + "BigBirdPegasusForSequenceClassification": { "tokenizer_classes": [ - "LayoutLMTokenizerFast", - "LayoutLMTokenizer" + "PegasusTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BigBirdPegasusForSequenceClassification" + ], + "sha": "a451bdb3f36fb76af06a51274d19fd88729443e6" }, - "LayoutLMv2Model": { + "BigBirdPegasusModel": { "tokenizer_classes": [ - "LayoutLMv2TokenizerFast", - "LayoutLMv2Tokenizer" + "PegasusTokenizerFast" ], - "processor_classes": [ - "LayoutLMv2ImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BigBirdPegasusModel" + ], + "sha": "3ece62a543ced3755e5ae239bcc3a4a5a6b75dd4" }, - "LayoutLMv3Model": { + "BioGptForCausalLM": { "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "BioGptTokenizer" ], - "processor_classes": [ - "LayoutLMv3ImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "BioGptForCausalLM" + ], + "sha": "07073b31da84054fd12226e3cae4cb3beb2547f9" }, - "LEDModel": { + "BioGptModel": { "tokenizer_classes": [ - "LEDTokenizerFast", - "LEDTokenizer" + "BioGptTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BioGptModel" + ], + "sha": "fe18551d0743538a990520b75707294ec57b4ebe" }, - "LevitModel": { + "BitBackbone": { "tokenizer_classes": [], "processor_classes": [ - "LevitImageProcessor" - ] - }, - "LiltModel": { - "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "BitImageProcessor" ], - "processor_classes": [] - }, - "LongformerModel": { - "tokenizer_classes": [ - "LongformerTokenizerFast", - "LongformerTokenizer" + "model_classes": [ + "BitBackbone" ], - "processor_classes": [] + "sha": "2f06f6b4395b6dce2b00ac839ff757410e743cd7" }, - "LongT5Model": { - "tokenizer_classes": [ - "T5TokenizerFast" + "BitForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "BitImageProcessor" ], - "processor_classes": [] - }, - "LukeModel": { - "tokenizer_classes": [ - "LukeTokenizer" + "model_classes": [ + "BitForImageClassification" ], - "processor_classes": [] + "sha": "d0d8476f2d285ddda7c42c0d4a8e4bf6f5d2bfdf" }, - "LxmertModel": { - "tokenizer_classes": [ - "LxmertTokenizerFast", - "LxmertTokenizer" + "BitModel": { + "tokenizer_classes": [], + "processor_classes": [ + "BitImageProcessor" ], - "processor_classes": [] + "model_classes": [ + "BitModel" + ], + "sha": "30a8a9b1a6b253cc500c01cf41bc1fc9581ea5e5" }, - "M2M100Model": { + "BlenderbotForCausalLM": { "tokenizer_classes": [ - "M2M100Tokenizer" + "BlenderbotTokenizer", + "BlenderbotTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BlenderbotForCausalLM" ], - "processor_classes": [] + "sha": "8aad2e13e8920bca3cf988ba45f8a7b008b51a81" }, - "MarianMTModel": { + "BlenderbotForConditionalGeneration": { "tokenizer_classes": [ - "MarianTokenizer" + "BlenderbotTokenizer", + "BlenderbotTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BlenderbotForConditionalGeneration", + "TFBlenderbotForConditionalGeneration" ], - "processor_classes": [] + "sha": "e8532878b9924fa02fb4b059b7f6e7fa372fff91" }, - "MarianModel": { + "BlenderbotModel": { "tokenizer_classes": [ - "MarianTokenizer" + "BlenderbotTokenizer", + "BlenderbotTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BlenderbotModel", + "TFBlenderbotModel" + ], + "sha": "ff848a40c30ca98eb7c6870bbb02677d5af9db55" }, - "MarkupLMModel": { + "BlenderbotSmallForCausalLM": { "tokenizer_classes": [ - "MarkupLMTokenizerFast", - "MarkupLMTokenizer" + "BlenderbotSmallTokenizer" ], - "processor_classes": [ - "MarkupLMFeatureExtractor" - ] - }, - "MaskFormerModel": { - "tokenizer_classes": [], - "processor_classes": [ - "MaskFormerFeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "BlenderbotSmallForCausalLM" + ], + "sha": "4c57c106630932eb9de4d76210a540d04616304d" }, - "MBartModel": { + "BlenderbotSmallForConditionalGeneration": { "tokenizer_classes": [ - "MBartTokenizerFast", - "MBartTokenizer" + "BlenderbotSmallTokenizer" ], - "processor_classes": [] - }, - "MCTCTModel": { - "tokenizer_classes": [], - "processor_classes": [ - "MCTCTFeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "BlenderbotSmallForConditionalGeneration", + "TFBlenderbotSmallForConditionalGeneration" + ], + "sha": "b8db01fcf3e37a5b369cd50e169bf383b8e905d8" }, - "MegatronBertModel": { + "BlenderbotSmallModel": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BlenderbotSmallTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "BlenderbotSmallModel", + "TFBlenderbotSmallModel" ], - "processor_classes": [] + "sha": "0a10c70e225ec63278faffa8fabf759f063f0e55" }, - "MobileBertModel": { + "Blip2ForConditionalGeneration": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" ], - "processor_classes": [] - }, - "MobileNetV2Model": { - "tokenizer_classes": [], "processor_classes": [ - "MobileNetV2ImageProcessor" - ] + "BlipImageProcessor" + ], + "model_classes": [ + "Blip2ForConditionalGeneration" + ], + "sha": "35e1ef43da3554af62eb29a7b3dbbef3f3bef48e" }, - "MobileViTModel": { - "tokenizer_classes": [], + "Blip2Model": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], "processor_classes": [ - "MobileViTImageProcessor" - ] + "BlipImageProcessor" + ], + "model_classes": [ + "Blip2Model" + ], + "sha": "c23378f225be31872fff33c103cf0ebc2454ffcc" }, - "MPNetModel": { + "BlipForConditionalGeneration": { "tokenizer_classes": [ - "MPNetTokenizerFast", - "MPNetTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [ + "BlipImageProcessor" + ], + "model_classes": [ + "BlipForConditionalGeneration" + ], + "sha": "e776bae5de3a4e9c11170b2465775eb37baf2bfe" }, - "MvpModel": { + "BlipModel": { "tokenizer_classes": [ - "MvpTokenizerFast", - "MvpTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [ + "BlipImageProcessor" + ], + "model_classes": [ + "BlipModel" ], - "processor_classes": [] + "sha": "261433f322f7146b0c28c0c025e92b3a33f716bb" }, - "NezhaModel": { + "BloomForCausalLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BloomTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BloomForCausalLM" ], - "processor_classes": [] + "sha": "0f4f06f162cd67d34d03ee156484e4001d468500" }, - "NystromformerModel": { + "BloomForQuestionAnswering": { "tokenizer_classes": [ - "AlbertTokenizerFast" + "BloomTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BloomForQuestionAnswering" ], - "processor_classes": [] + "sha": "23f369f163eef8c9c9685900440b0cbb0f3439fd" }, - "OpenAIGPTLMHeadModel": { + "BloomForSequenceClassification": { "tokenizer_classes": [ - "OpenAIGPTTokenizerFast", - "OpenAIGPTTokenizer" + "BloomTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BloomForSequenceClassification" ], - "processor_classes": [] + "sha": "b2280eef7172835f39b265eb0c46623257f67bbe" }, - "OpenAIGPTModel": { + "BloomForTokenClassification": { "tokenizer_classes": [ - "OpenAIGPTTokenizerFast", - "OpenAIGPTTokenizer" + "BloomTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "BloomForTokenClassification" + ], + "sha": "9796aa45f99adff987c978089e11c0bd9d7b997f" }, - "OPTModel": { + "BloomModel": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "BloomTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "BloomModel" ], - "processor_classes": [] + "sha": "28b600fcfdc4f4938406fb518abf895620048cb2" }, - "OwlViTModel": { + "CLIPModel": { "tokenizer_classes": [ - "CLIPTokenizerFast", - "CLIPTokenizer" + "CLIPTokenizer", + "CLIPTokenizerFast" ], "processor_classes": [ - "OwlViTFeatureExtractor" - ] - }, - "PegasusModel": { - "tokenizer_classes": [ - "PegasusTokenizerFast" + "CLIPImageProcessor" ], - "processor_classes": [] - }, - "PegasusXModel": { - "tokenizer_classes": [ - "PegasusTokenizerFast" + "model_classes": [ + "CLIPModel", + "TFCLIPModel" ], - "processor_classes": [] + "sha": "0452d344074485d0e7eb5d5c12447b7c9dbc9619" }, - "PerceiverModel": { + "CLIPSegModel": { "tokenizer_classes": [ - "PerceiverTokenizer" + "CLIPTokenizer", + "CLIPTokenizerFast" + ], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "CLIPSegModel" ], - "processor_classes": [] + "sha": "7b1305214ccc85d29b776ffbee06748693852a04" }, - "PLBartModel": { + "CTRLForSequenceClassification": { "tokenizer_classes": [ - "PLBartTokenizer" + "CTRLTokenizer" ], - "processor_classes": [] - }, - "PoolFormerModel": { - "tokenizer_classes": [], - "processor_classes": [ - "PoolFormerImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "CTRLForSequenceClassification", + "TFCTRLForSequenceClassification" + ], + "sha": "280b5a3502d607c55c9f8d9f198fe9c2802d6f73" }, - "ProphetNetModel": { + "CTRLLMHeadModel": { "tokenizer_classes": [ - "ProphetNetTokenizer" + "CTRLTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "CTRLLMHeadModel", + "TFCTRLLMHeadModel" ], - "processor_classes": [] + "sha": "662381663b216f1dd3c9cd30e2e83cb4c6fc9552" }, - "ReformerModel": { + "CTRLModel": { "tokenizer_classes": [ - "ReformerTokenizerFast", - "ReformerTokenizer" + "CTRLTokenizer" ], - "processor_classes": [] - }, - "RegNetModel": { - "tokenizer_classes": [], - "processor_classes": [ - "ConvNextImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "CTRLModel", + "TFCTRLModel" + ], + "sha": "68b19b4f132d5a191a73acd78d983cbdcf068e9c" }, - "RemBertModel": { + "CanineForMultipleChoice": { "tokenizer_classes": [ - "RemBertTokenizerFast" + "CanineTokenizer" ], - "processor_classes": [] - }, - "ResNetModel": { - "tokenizer_classes": [], - "processor_classes": [ - "ConvNextImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "CanineForMultipleChoice" + ], + "sha": "fa0451453ed202f903ff7dcf6071aab6630fb89f" }, - "RobertaModel": { + "CanineForQuestionAnswering": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "CanineTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "CanineForQuestionAnswering" ], - "processor_classes": [] + "sha": "5e1012bb086ac2e0b1497eeb7ed14eb2183d4ecb" }, - "RoCBertModel": { + "CanineForSequenceClassification": { "tokenizer_classes": [ - "RoCBertTokenizer" + "CanineTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "CanineForSequenceClassification" + ], + "sha": "75336dc9179153869c38a8047ce4b1e02677a260" }, - "RoFormerModel": { + "CanineForTokenClassification": { "tokenizer_classes": [ - "RoFormerTokenizerFast", - "RoFormerTokenizer" + "CanineTokenizer" ], - "processor_classes": [] - }, - "SegformerModel": { - "tokenizer_classes": [], - "processor_classes": [ - "SegformerImageProcessor" - ] + "processor_classes": [], + "model_classes": [ + "CanineForTokenClassification" + ], + "sha": "65a622ea8e12597e12f45e59d46d8dbe8461fc10" }, - "SEWDModel": { + "CanineModel": { "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" + "CanineTokenizer" ], - "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "CanineModel" + ], + "sha": "531ef67ad4f0b3dc7a9e5d722c774096b7401b1b" }, - "SEWModel": { + "ChineseCLIPModel": { "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "ChineseCLIPImageProcessor" + ], + "model_classes": [ + "ChineseCLIPModel" + ], + "sha": "504271a3c5fd9c2e877f5b4c01848bc18778c7c3" }, - "Speech2TextModel": { + "CodeGenForCausalLM": { "tokenizer_classes": [ - "Speech2TextTokenizer" + "CodeGenTokenizer", + "CodeGenTokenizerFast" ], - "processor_classes": [ - "Speech2TextFeatureExtractor" - ] - }, - "SplinterModel": { - "tokenizer_classes": [], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "CodeGenForCausalLM" + ], + "sha": "a3fc69d757fd1f0aa01bcbc4337f586651c7cb10" }, - "SqueezeBertModel": { + "CodeGenModel": { "tokenizer_classes": [ - "SqueezeBertTokenizerFast", - "SqueezeBertTokenizer" + "CodeGenTokenizer", + "CodeGenTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "CodeGenModel" ], - "processor_classes": [] + "sha": "dad4941a2b7429fc6e8206fcc4a04fc40f4a0beb" }, - "SwinModel": { + "ConditionalDetrForObjectDetection": { "tokenizer_classes": [], "processor_classes": [ - "ViTImageProcessor" - ] + "ConditionalDetrImageProcessor" + ], + "model_classes": [ + "ConditionalDetrForObjectDetection" + ], + "sha": "762c213a0285edc84eb813a2ed90063cf971ca43" }, - "Swinv2Model": { + "ConditionalDetrModel": { "tokenizer_classes": [], "processor_classes": [ - "ViTImageProcessor" - ] - }, - "SwitchTransformersModel": { - "tokenizer_classes": [ - "T5TokenizerFast" + "ConditionalDetrImageProcessor" ], - "processor_classes": [] + "model_classes": [ + "ConditionalDetrModel" + ], + "sha": "18b75874158cac520c63605293b06e0b1327c263" }, - "T5Model": { + "ConvBertForMaskedLM": { "tokenizer_classes": [ - "T5TokenizerFast" + "ConvBertTokenizer", + "ConvBertTokenizerFast" ], - "processor_classes": [] - }, - "TableTransformerModel": { - "tokenizer_classes": [], - "processor_classes": [ - "DetrFeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "ConvBertForMaskedLM", + "TFConvBertForMaskedLM" + ], + "sha": "307c70e32c3d3c18aeb45e0cbdc9fcd2957d9aba" }, - "TapasModel": { + "ConvBertForMultipleChoice": { "tokenizer_classes": [ - "TapasTokenizer" + "ConvBertTokenizer", + "ConvBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ConvBertForMultipleChoice", + "TFConvBertForMultipleChoice" + ], + "sha": "d6561a21ffdb82d03c1822af0510eb7482ce5026" }, - "TransfoXLLMHeadModel": { + "ConvBertForQuestionAnswering": { "tokenizer_classes": [ - "TransfoXLTokenizer" + "ConvBertTokenizer", + "ConvBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ConvBertForQuestionAnswering", + "TFConvBertForQuestionAnswering" ], - "processor_classes": [] + "sha": "8a056da5cc421415c2a24b9f644dd95ca279411d" }, - "TransfoXLModel": { + "ConvBertForSequenceClassification": { "tokenizer_classes": [ - "TransfoXLTokenizer" + "ConvBertTokenizer", + "ConvBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ConvBertForSequenceClassification", + "TFConvBertForSequenceClassification" ], - "processor_classes": [] + "sha": "8bb8b20e51d282d777cc567cacadd97a35f0811e" }, - "UniSpeechSatModel": { + "ConvBertForTokenClassification": { "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" + "ConvBertTokenizer", + "ConvBertTokenizerFast" ], - "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "ConvBertForTokenClassification", + "TFConvBertForTokenClassification" + ], + "sha": "8db0dd3c2b8ccc958fa9a84801f4f837b42fcf2c" }, - "UniSpeechModel": { + "ConvBertModel": { "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" + "ConvBertTokenizer", + "ConvBertTokenizerFast" ], - "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "ConvBertModel", + "TFConvBertModel" + ], + "sha": "c9c5b1a74f0e468d8467473cabeaa67fcdbaddb7" }, - "VanModel": { + "ConvNextBackbone": { "tokenizer_classes": [], "processor_classes": [ "ConvNextImageProcessor" - ] + ], + "model_classes": [ + "ConvNextBackbone" + ], + "sha": "499c7d6a97825b79e19663b70f3b60c4813b6bf2" }, - "VideoMAEModel": { + "ConvNextForImageClassification": { "tokenizer_classes": [], "processor_classes": [ - "VideoMAEImageProcessor" - ] - }, - "ViltModel": { - "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "ConvNextImageProcessor" ], - "processor_classes": [ - "ViltImageProcessor" - ] - }, - "VisualBertModel": { - "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "model_classes": [ + "ConvNextForImageClassification", + "TFConvNextForImageClassification" ], - "processor_classes": [] + "sha": "0b490fd6b19cdbf721025dbd6ee45dcc5828e6e3" }, - "ViTModel": { + "ConvNextModel": { "tokenizer_classes": [], "processor_classes": [ - "ViTImageProcessor" - ] + "ConvNextImageProcessor" + ], + "model_classes": [ + "ConvNextModel", + "TFConvNextModel" + ], + "sha": "7b3b47a57b9a9120e022b91d6067daeac55b794f" }, - "ViTMAEModel": { + "ConvNextV2Backbone": { "tokenizer_classes": [], "processor_classes": [ - "ViTImageProcessor" - ] + "ConvNextImageProcessor" + ], + "model_classes": [ + "ConvNextV2Backbone" + ], + "sha": "c82fc526949dfd892a1fee3c34be6f8d80c4d3df" }, - "ViTMSNModel": { + "ConvNextV2ForImageClassification": { "tokenizer_classes": [], "processor_classes": [ - "ViTImageProcessor" - ] - }, - "Wav2Vec2ConformerModel": { - "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" + "ConvNextImageProcessor" ], - "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] - }, - "Wav2Vec2Model": { - "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" + "model_classes": [ + "ConvNextV2ForImageClassification" ], - "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "sha": "ee22bae1cbb87d66fc7f62f7e15a43d6ff80d3cc" }, - "WavLMModel": { - "tokenizer_classes": [ - "Wav2Vec2CTCTokenizer" - ], + "ConvNextV2Model": { + "tokenizer_classes": [], "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] - }, - "WhisperModel": { - "tokenizer_classes": [ - "WhisperTokenizer" + "ConvNextImageProcessor" ], - "processor_classes": [ - "WhisperFeatureExtractor" - ] - }, - "XCLIPModel": { - "tokenizer_classes": [ - "CLIPTokenizerFast", - "CLIPTokenizer" + "model_classes": [ + "ConvNextV2Model" ], - "processor_classes": [ - "VideoMAEImageProcessor", - "CLIPImageProcessor" - ] + "sha": "c4dd68ee1102cba05bcc483da2a88e39427b7249" }, - "XGLMModel": { - "tokenizer_classes": [ - "XGLMTokenizerFast" + "CvtForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "ConvNextImageProcessor" ], - "processor_classes": [] - }, - "XLMRobertaXLModel": { - "tokenizer_classes": [ - "XLMRobertaTokenizerFast" + "model_classes": [ + "CvtForImageClassification", + "TFCvtForImageClassification" ], - "processor_classes": [] + "sha": "4b1938e252fdb26a06c1f5755e07fa8f6eed2d75" }, - "XLMModel": { - "tokenizer_classes": [ - "XLMTokenizer" + "CvtModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ConvNextImageProcessor" ], - "processor_classes": [] - }, - "XLMWithLMHeadModel": { - "tokenizer_classes": [ - "XLMTokenizer" + "model_classes": [ + "CvtModel", + "TFCvtModel" ], - "processor_classes": [] + "sha": "27fed12c174f4f4f1fe27075d1c29602fe0669f0" }, - "XLNetLMHeadModel": { + "DPRQuestionEncoder": { "tokenizer_classes": [ - "XLNetTokenizerFast", - "XLNetTokenizer" + "DPRQuestionEncoderTokenizer", + "DPRQuestionEncoderTokenizerFast" ], - "processor_classes": [] - }, - "XLNetModel": { - "tokenizer_classes": [ - "XLNetTokenizerFast", - "XLNetTokenizer" + "processor_classes": [], + "model_classes": [ + "DPRQuestionEncoder", + "TFDPRQuestionEncoder" ], - "processor_classes": [] + "sha": "09ae0269780271e0a4916f7bab1dbc4f8a76070d" }, - "YolosModel": { + "DPTForDepthEstimation": { "tokenizer_classes": [], "processor_classes": [ - "YolosFeatureExtractor" - ] - }, - "YosoModel": { - "tokenizer_classes": [ - "AlbertTokenizerFast" - ], - "processor_classes": [] - }, - "AlbertForMaskedLM": { - "tokenizer_classes": [ - "AlbertTokenizerFast" + "DPTImageProcessor" ], - "processor_classes": [] - }, - "AlbertForMultipleChoice": { - "tokenizer_classes": [ - "AlbertTokenizerFast" + "model_classes": [ + "DPTForDepthEstimation" ], - "processor_classes": [] + "sha": "11b7735d64d95b6599811631b012d2dec6eaa2c1" }, - "AlbertForPreTraining": { - "tokenizer_classes": [ - "AlbertTokenizerFast" + "DPTForSemanticSegmentation": { + "tokenizer_classes": [], + "processor_classes": [ + "DPTImageProcessor" ], - "processor_classes": [] - }, - "AlbertForQuestionAnswering": { - "tokenizer_classes": [ - "AlbertTokenizerFast" + "model_classes": [ + "DPTForSemanticSegmentation" ], - "processor_classes": [] + "sha": "e140c3c716a4bf11dad875e5f5f0abd2bd4cbbcb" }, - "AlbertForSequenceClassification": { - "tokenizer_classes": [ - "AlbertTokenizerFast" + "DPTModel": { + "tokenizer_classes": [], + "processor_classes": [ + "DPTImageProcessor" ], - "processor_classes": [] - }, - "AlbertForTokenClassification": { - "tokenizer_classes": [ - "AlbertTokenizerFast" + "model_classes": [ + "DPTModel" ], - "processor_classes": [] + "sha": "1d6ae6c0b60868dffbef0dddeda381c51c6dcba5" }, - "BartForCausalLM": { - "tokenizer_classes": [ - "BartTokenizerFast", - "BartTokenizer" + "Data2VecAudioForCTC": { + "tokenizer_classes": [], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" ], - "processor_classes": [] - }, - "BartForConditionalGeneration": { - "tokenizer_classes": [ - "BartTokenizerFast", - "BartTokenizer" + "model_classes": [ + "Data2VecAudioForCTC" ], - "processor_classes": [] + "sha": "bb161b6a181bd2c22cf30222f46fa6ef42225744" }, - "BartForQuestionAnswering": { - "tokenizer_classes": [ - "BartTokenizerFast", - "BartTokenizer" + "Data2VecAudioForSequenceClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" ], - "processor_classes": [] - }, - "BartForSequenceClassification": { - "tokenizer_classes": [ - "BartTokenizerFast", - "BartTokenizer" + "model_classes": [ + "Data2VecAudioForSequenceClassification" ], - "processor_classes": [] + "sha": "8de17e0a959eca5f72b2ea59a11bc1fa744785d9" }, - "BeitForImageClassification": { + "Data2VecAudioForXVector": { "tokenizer_classes": [], "processor_classes": [ - "BeitImageProcessor" - ] + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "Data2VecAudioForXVector" + ], + "sha": "dcb92484cf28fb4fe1dcf5d6e8d78e04382fdce9" }, - "BeitForSemanticSegmentation": { + "Data2VecAudioModel": { "tokenizer_classes": [], "processor_classes": [ - "BeitImageProcessor" - ] - }, - "BertForMaskedLM": { - "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "Wav2Vec2FeatureExtractor" ], - "processor_classes": [] - }, - "BertForMultipleChoice": { - "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "model_classes": [ + "Data2VecAudioModel" ], - "processor_classes": [] + "sha": "73f503fdff73b7616154f64dbe38a685cc48e8eb" }, - "BertForNextSentencePrediction": { + "Data2VecTextForCausalLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] - }, - "BertForPreTraining": { - "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "processor_classes": [], + "model_classes": [ + "Data2VecTextForCausalLM" ], - "processor_classes": [] + "sha": "1f3658ce623653338cd31516551e8181aa08bb38" }, - "BertForQuestionAnswering": { + "Data2VecTextForMaskedLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] - }, - "BertForSequenceClassification": { - "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "processor_classes": [], + "model_classes": [ + "Data2VecTextForMaskedLM" ], - "processor_classes": [] + "sha": "fb41ac30d0faa0899bf5afaa0986df8993395ca6" }, - "BertForTokenClassification": { + "Data2VecTextForMultipleChoice": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] - }, - "BigBirdForCausalLM": { - "tokenizer_classes": [ - "BigBirdTokenizerFast" + "processor_classes": [], + "model_classes": [ + "Data2VecTextForMultipleChoice" ], - "processor_classes": [] + "sha": "e7556d520ad90ebae5ad88554d45a37488d00040" }, - "BigBirdForMaskedLM": { + "Data2VecTextForQuestionAnswering": { "tokenizer_classes": [ - "BigBirdTokenizerFast" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] - }, - "BigBirdForMultipleChoice": { - "tokenizer_classes": [ - "BigBirdTokenizerFast" + "processor_classes": [], + "model_classes": [ + "Data2VecTextForQuestionAnswering" ], - "processor_classes": [] + "sha": "9630833d76a1fd7e96b904d87bb11b7c00ccd021" }, - "BigBirdForPreTraining": { + "Data2VecTextForSequenceClassification": { "tokenizer_classes": [ - "BigBirdTokenizerFast" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] - }, - "BigBirdForQuestionAnswering": { - "tokenizer_classes": [ - "BigBirdTokenizerFast" + "processor_classes": [], + "model_classes": [ + "Data2VecTextForSequenceClassification" ], - "processor_classes": [] + "sha": "156e4019c37d9592f193ba80553cd245cbccecb3" }, - "BigBirdForSequenceClassification": { + "Data2VecTextForTokenClassification": { "tokenizer_classes": [ - "BigBirdTokenizerFast" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] - }, - "BigBirdForTokenClassification": { - "tokenizer_classes": [ - "BigBirdTokenizerFast" + "processor_classes": [], + "model_classes": [ + "Data2VecTextForTokenClassification" ], - "processor_classes": [] + "sha": "55b3a49fdbf22479d6eb939261d4b884ea288270" }, - "BigBirdPegasusForCausalLM": { + "Data2VecTextModel": { "tokenizer_classes": [ - "PegasusTokenizerFast" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] - }, - "BigBirdPegasusForConditionalGeneration": { - "tokenizer_classes": [ - "PegasusTokenizerFast" + "processor_classes": [], + "model_classes": [ + "Data2VecTextModel" ], - "processor_classes": [] + "sha": "c21be3e4f88e8357bf33bfba8f8e05ae2e735124" }, - "BigBirdPegasusForQuestionAnswering": { - "tokenizer_classes": [ - "PegasusTokenizerFast" + "Data2VecVisionForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "BeitImageProcessor" ], - "processor_classes": [] - }, - "BigBirdPegasusForSequenceClassification": { - "tokenizer_classes": [ - "PegasusTokenizerFast" + "model_classes": [ + "Data2VecVisionForImageClassification", + "TFData2VecVisionForImageClassification" ], - "processor_classes": [] + "sha": "d640e7ced7a3fbbb8c8661a4f67b934e55406172" }, - "BlenderbotForCausalLM": { - "tokenizer_classes": [ - "BlenderbotTokenizerFast", - "BlenderbotTokenizer" + "Data2VecVisionForSemanticSegmentation": { + "tokenizer_classes": [], + "processor_classes": [ + "BeitImageProcessor" ], - "processor_classes": [] - }, - "BlenderbotForConditionalGeneration": { - "tokenizer_classes": [ - "BlenderbotTokenizerFast", - "BlenderbotTokenizer" + "model_classes": [ + "Data2VecVisionForSemanticSegmentation", + "TFData2VecVisionForSemanticSegmentation" ], - "processor_classes": [] + "sha": "3eba3cd694fab6530b7e5da8f49d3951301c816a" }, - "BlenderbotSmallForCausalLM": { + "Data2VecVisionModel": { "tokenizer_classes": [], - "processor_classes": [] + "processor_classes": [ + "BeitImageProcessor" + ], + "model_classes": [ + "Data2VecVisionModel", + "TFData2VecVisionModel" + ], + "sha": "2a7ad25e4359970dc70494a2f3eb98e2a3c9806d" }, - "BlenderbotSmallForConditionalGeneration": { - "tokenizer_classes": [], - "processor_classes": [] + "DebertaForMaskedLM": { + "tokenizer_classes": [ + "DebertaTokenizer", + "DebertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DebertaForMaskedLM", + "TFDebertaForMaskedLM" + ], + "sha": "e0f9ada9e0f6d4d7cc39d7cbd58369b0c84de33d" }, - "BloomForCausalLM": { + "DebertaForQuestionAnswering": { "tokenizer_classes": [ - "BloomTokenizerFast" + "DebertaTokenizer", + "DebertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "DebertaForQuestionAnswering", + "TFDebertaForQuestionAnswering" + ], + "sha": "a3eb69cdb0b52f7d0fb730e882f1a54b9a7442ea" }, - "BloomForQuestionAnswering": { + "DebertaForSequenceClassification": { "tokenizer_classes": [ - "BloomTokenizerFast" + "DebertaTokenizer", + "DebertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "DebertaForSequenceClassification", + "TFDebertaForSequenceClassification" + ], + "sha": "32af91d12c4e9b6d62b420bee93311fd77d3c933" }, - "BloomForSequenceClassification": { + "DebertaForTokenClassification": { "tokenizer_classes": [ - "BloomTokenizerFast" + "DebertaTokenizer", + "DebertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "DebertaForTokenClassification", + "TFDebertaForTokenClassification" + ], + "sha": "ba62ba2726d813e60e512476fc1b178aa3858175" }, - "BloomForTokenClassification": { + "DebertaModel": { "tokenizer_classes": [ - "BloomTokenizerFast" + "DebertaTokenizer", + "DebertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "DebertaModel", + "TFDebertaModel" + ], + "sha": "4273294e14cd04c0e2cd1dcff5cf7e5d4fe906ba" }, - "CTRLForSequenceClassification": { + "DebertaV2ForMaskedLM": { "tokenizer_classes": [ - "CTRLTokenizer" + "DebertaV2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DebertaV2ForMaskedLM", + "TFDebertaV2ForMaskedLM" ], - "processor_classes": [] + "sha": "9089b6afa8f66fd16503fca2b7c54b50c2195123" }, - "CanineForMultipleChoice": { + "DebertaV2ForMultipleChoice": { "tokenizer_classes": [ - "CanineTokenizer" + "DebertaV2TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "DebertaV2ForMultipleChoice" + ], + "sha": "7acf4b7415b2869e225a5ed82e68f1c0374e9668" }, - "CanineForQuestionAnswering": { + "DebertaV2ForQuestionAnswering": { "tokenizer_classes": [ - "CanineTokenizer" + "DebertaV2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DebertaV2ForQuestionAnswering", + "TFDebertaV2ForQuestionAnswering" ], - "processor_classes": [] + "sha": "17ef18fefddc0ec61a972eea06af430059ae3759" }, - "CanineForSequenceClassification": { + "DebertaV2ForSequenceClassification": { "tokenizer_classes": [ - "CanineTokenizer" + "DebertaV2TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "DebertaV2ForSequenceClassification", + "TFDebertaV2ForSequenceClassification" + ], + "sha": "1ef484d43eb15ac6b1f8be393c3d59bea2267dc9" }, - "CanineForTokenClassification": { + "DebertaV2ForTokenClassification": { "tokenizer_classes": [ - "CanineTokenizer" + "DebertaV2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DebertaV2ForTokenClassification", + "TFDebertaV2ForTokenClassification" ], - "processor_classes": [] + "sha": "5c4e629b5b03957a546f7f76c31f6887f99fc17c" }, - "CodeGenForCausalLM": { + "DebertaV2Model": { "tokenizer_classes": [ - "CodeGenTokenizerFast", - "CodeGenTokenizer" + "DebertaV2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DebertaV2Model", + "TFDebertaV2Model" ], - "processor_classes": [] + "sha": "a1945cc2bb1ef207f8fdeb4ea146711ade1db77a" }, - "ConditionalDetrForObjectDetection": { + "DeformableDetrForObjectDetection": { "tokenizer_classes": [], "processor_classes": [ - "ConditionalDetrFeatureExtractor" - ] - }, - "ConvBertForMaskedLM": { - "tokenizer_classes": [ - "ConvBertTokenizerFast", - "ConvBertTokenizer" + "DeformableDetrImageProcessor" ], - "processor_classes": [] - }, - "ConvBertForMultipleChoice": { - "tokenizer_classes": [ - "ConvBertTokenizerFast", - "ConvBertTokenizer" + "model_classes": [ + "DeformableDetrForObjectDetection" ], - "processor_classes": [] + "sha": "8fa0db215c458f60ae4d455d6fb067c1c5e39fdc" }, - "ConvBertForQuestionAnswering": { - "tokenizer_classes": [ - "ConvBertTokenizerFast", - "ConvBertTokenizer" + "DeformableDetrModel": { + "tokenizer_classes": [], + "processor_classes": [ + "DeformableDetrImageProcessor" ], - "processor_classes": [] - }, - "ConvBertForSequenceClassification": { - "tokenizer_classes": [ - "ConvBertTokenizerFast", - "ConvBertTokenizer" + "model_classes": [ + "DeformableDetrModel" ], - "processor_classes": [] + "sha": "0faac5624696b03edd14694642f9804f2cd8f3da" }, - "ConvBertForTokenClassification": { - "tokenizer_classes": [ - "ConvBertTokenizerFast", - "ConvBertTokenizer" + "DeiTForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "DeiTImageProcessor" ], - "processor_classes": [] + "model_classes": [ + "DeiTForImageClassification", + "TFDeiTForImageClassification" + ], + "sha": "21fc864199dafa0130f16a45769c6b6ca22c7784" }, - "ConvNextForImageClassification": { + "DeiTForImageClassificationWithTeacher": { "tokenizer_classes": [], "processor_classes": [ - "ConvNextImageProcessor" - ] + "DeiTImageProcessor" + ], + "model_classes": [ + "DeiTForImageClassificationWithTeacher", + "TFDeiTForImageClassificationWithTeacher" + ], + "sha": "5a5738a109e27f3d4b78a0db4cb1d3331140c10e" }, - "CvtForImageClassification": { + "DeiTForMaskedImageModeling": { "tokenizer_classes": [], "processor_classes": [ - "ConvNextImageProcessor" - ] + "DeiTImageProcessor" + ], + "model_classes": [ + "DeiTForMaskedImageModeling", + "TFDeiTForMaskedImageModeling" + ], + "sha": "d5df5c538fe1efb8d668a3893d1691d505a0de06" }, - "DPTForDepthEstimation": { + "DeiTModel": { "tokenizer_classes": [], "processor_classes": [ - "DPTImageProcessor" - ] + "DeiTImageProcessor" + ], + "model_classes": [ + "DeiTModel", + "TFDeiTModel" + ], + "sha": "0fdbff6f44b7c6933c2027fec1d7f87bec06b590" }, - "DPTForSemanticSegmentation": { + "DetaForObjectDetection": { "tokenizer_classes": [], "processor_classes": [ - "DPTImageProcessor" - ] + "DetaImageProcessor" + ], + "model_classes": [ + "DetaForObjectDetection" + ], + "sha": "a15ad6ce64fbcb5021b2b99e9587c4011ef3341d" }, - "Data2VecAudioForCTC": { + "DetaModel": { "tokenizer_classes": [], "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "DetaImageProcessor" + ], + "model_classes": [ + "DetaModel" + ], + "sha": "8820f2297ec0dec8f1875054559c8b7a162098e3" }, - "Data2VecAudioForSequenceClassification": { + "DetrForObjectDetection": { "tokenizer_classes": [], "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] + "DetrImageProcessor" + ], + "model_classes": [ + "DetrForObjectDetection" + ], + "sha": "7dc967c53f4b3f07904c42b255346b744d0ad84e" }, - "Data2VecAudioForXVector": { + "DetrForSegmentation": { "tokenizer_classes": [], "processor_classes": [ - "Wav2Vec2FeatureExtractor" - ] - }, - "Data2VecTextForCausalLM": { - "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "DetrImageProcessor" ], - "processor_classes": [] - }, - "Data2VecTextForMaskedLM": { - "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "model_classes": [ + "DetrForSegmentation" ], - "processor_classes": [] + "sha": "e34330acdae359588ef853e961a78d419dc4e8eb" }, - "Data2VecTextForMultipleChoice": { - "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "DetrModel": { + "tokenizer_classes": [], + "processor_classes": [ + "DetrImageProcessor" ], - "processor_classes": [] - }, - "Data2VecTextForQuestionAnswering": { - "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "model_classes": [ + "DetrModel" ], - "processor_classes": [] + "sha": "f15ce38a10c7447e8048b1681e4811322a005722" }, - "Data2VecTextForSequenceClassification": { - "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "DinatBackbone": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" ], - "processor_classes": [] - }, - "Data2VecTextForTokenClassification": { - "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "model_classes": [ + "DinatBackbone" ], - "processor_classes": [] + "sha": "3ba13790a0796d90104c207f75bb3d5d79723d51" }, - "Data2VecVisionForImageClassification": { + "DinatForImageClassification": { "tokenizer_classes": [], "processor_classes": [ - "BeitImageProcessor" - ] + "ViTImageProcessor" + ], + "model_classes": [ + "DinatForImageClassification" + ], + "sha": "624cf2d864a7ea2f90e24014a213e34597e8bd76" }, - "Data2VecVisionForSemanticSegmentation": { + "DinatModel": { "tokenizer_classes": [], "processor_classes": [ - "BeitImageProcessor" - ] - }, - "DebertaForMaskedLM": { - "tokenizer_classes": [ - "DebertaTokenizerFast", - "DebertaTokenizer" + "ViTImageProcessor" ], - "processor_classes": [] - }, - "DebertaForQuestionAnswering": { - "tokenizer_classes": [ - "DebertaTokenizerFast", - "DebertaTokenizer" + "model_classes": [ + "DinatModel" ], - "processor_classes": [] + "sha": "d6c75bc51196f0a683afb12de6310fdda13efefd" }, - "DebertaForSequenceClassification": { + "DistilBertForMaskedLM": { "tokenizer_classes": [ - "DebertaTokenizerFast", - "DebertaTokenizer" + "DistilBertTokenizer", + "DistilBertTokenizerFast" ], - "processor_classes": [] - }, - "DebertaForTokenClassification": { - "tokenizer_classes": [ - "DebertaTokenizerFast", - "DebertaTokenizer" + "processor_classes": [], + "model_classes": [ + "DistilBertForMaskedLM", + "TFDistilBertForMaskedLM" ], - "processor_classes": [] + "sha": "b2dfda30b012821996e6e603729562d9c900bc0f" }, - "DebertaV2ForMaskedLM": { + "DistilBertForMultipleChoice": { "tokenizer_classes": [ - "DebertaV2TokenizerFast" + "DistilBertTokenizer", + "DistilBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DistilBertForMultipleChoice", + "TFDistilBertForMultipleChoice" ], - "processor_classes": [] + "sha": "ec6b83129a7d1be2a6b8d58303abcca5541a5cb3" }, - "DebertaV2ForMultipleChoice": { + "DistilBertForQuestionAnswering": { "tokenizer_classes": [ - "DebertaV2TokenizerFast" + "DistilBertTokenizer", + "DistilBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DistilBertForQuestionAnswering", + "TFDistilBertForQuestionAnswering" ], - "processor_classes": [] + "sha": "812406b226415044469b0e0a84c4fe0ff338c5d3" }, - "DebertaV2ForQuestionAnswering": { + "DistilBertForSequenceClassification": { "tokenizer_classes": [ - "DebertaV2TokenizerFast" + "DistilBertTokenizer", + "DistilBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DistilBertForSequenceClassification", + "TFDistilBertForSequenceClassification" ], - "processor_classes": [] + "sha": "6f427ce7b3e5aaa596938fbd98437d3875581b7b" }, - "DebertaV2ForSequenceClassification": { + "DistilBertForTokenClassification": { "tokenizer_classes": [ - "DebertaV2TokenizerFast" + "DistilBertTokenizer", + "DistilBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "DistilBertForTokenClassification", + "TFDistilBertForTokenClassification" ], - "processor_classes": [] + "sha": "166dbe3f5d6ecd871762567069454d6ec65234b4" }, - "DebertaV2ForTokenClassification": { + "DistilBertModel": { "tokenizer_classes": [ - "DebertaV2TokenizerFast" + "DistilBertTokenizer", + "DistilBertTokenizerFast" ], - "processor_classes": [] - }, - "DeformableDetrForObjectDetection": { - "tokenizer_classes": [], - "processor_classes": [ - "DeformableDetrFeatureExtractor" - ] + "processor_classes": [], + "model_classes": [ + "DistilBertModel", + "TFDistilBertModel" + ], + "sha": "cc4425ad0676f3ec00e8bffe485fe83cae61041a" }, - "DeiTForImageClassification": { + "DonutSwinModel": { "tokenizer_classes": [], "processor_classes": [ - "DeiTImageProcessor" - ] + "DonutImageProcessor" + ], + "model_classes": [ + "DonutSwinModel" + ], + "sha": "1b10654fbfe2f2ea410a672ab605bd5c60d3f284" }, - "DeiTForImageClassificationWithTeacher": { + "EfficientFormerForImageClassification": { "tokenizer_classes": [], "processor_classes": [ - "DeiTImageProcessor" - ] + "EfficientFormerImageProcessor" + ], + "model_classes": [ + "EfficientFormerForImageClassification" + ], + "sha": "ebadb628e12f268e321fcc756fa4606f7b5b3178" }, - "DeiTForMaskedImageModeling": { + "EfficientFormerForImageClassificationWithTeacher": { "tokenizer_classes": [], "processor_classes": [ - "DeiTImageProcessor" - ] + "EfficientFormerImageProcessor" + ], + "model_classes": [ + "EfficientFormerForImageClassificationWithTeacher" + ], + "sha": "1beabce6da9cb4ebbeafcd1ef23fac36b4a269e2" }, - "DetrForObjectDetection": { + "EfficientFormerModel": { "tokenizer_classes": [], "processor_classes": [ - "DetrFeatureExtractor" - ] + "EfficientFormerImageProcessor" + ], + "model_classes": [ + "EfficientFormerModel" + ], + "sha": "200fae5b875844d09c8a91d1c155b72b06a517f6" }, - "DetrForSegmentation": { + "EfficientNetForImageClassification": { "tokenizer_classes": [], "processor_classes": [ - "DetrFeatureExtractor" - ] - }, - "DistilBertForMaskedLM": { - "tokenizer_classes": [ - "DistilBertTokenizerFast", - "DistilBertTokenizer" - ], - "processor_classes": [] - }, - "DistilBertForMultipleChoice": { - "tokenizer_classes": [ - "DistilBertTokenizerFast", - "DistilBertTokenizer" + "EfficientNetImageProcessor" ], - "processor_classes": [] - }, - "DistilBertForQuestionAnswering": { - "tokenizer_classes": [ - "DistilBertTokenizerFast", - "DistilBertTokenizer" + "model_classes": [ + "EfficientNetForImageClassification" ], - "processor_classes": [] + "sha": "6ed195ee636d2c0b885139da8c7b45d57ebaeee0" }, - "DistilBertForSequenceClassification": { - "tokenizer_classes": [ - "DistilBertTokenizerFast", - "DistilBertTokenizer" + "EfficientNetModel": { + "tokenizer_classes": [], + "processor_classes": [ + "EfficientNetImageProcessor" ], - "processor_classes": [] - }, - "DistilBertForTokenClassification": { - "tokenizer_classes": [ - "DistilBertTokenizerFast", - "DistilBertTokenizer" + "model_classes": [ + "EfficientNetModel" ], - "processor_classes": [] + "sha": "eb03c90d4aaad98af0f19e0dfbdc41106297ffff" }, "ElectraForCausalLM": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "ElectraTokenizer", + "ElectraTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ElectraForCausalLM" + ], + "sha": "c78396bc8cdd8db247892339de8da80d691d1d04" }, "ElectraForMaskedLM": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "ElectraTokenizer", + "ElectraTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ElectraForMaskedLM", + "TFElectraForMaskedLM" ], - "processor_classes": [] + "sha": "631337703dbd8d41904c39891a41c6f1edd31813" }, "ElectraForMultipleChoice": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "ElectraTokenizer", + "ElectraTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ElectraForMultipleChoice", + "TFElectraForMultipleChoice" + ], + "sha": "66fdea6e22cfcbd3caa49ea82f31871c460612fa" }, "ElectraForPreTraining": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "ElectraTokenizer", + "ElectraTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ElectraForPreTraining", + "TFElectraForPreTraining" ], - "processor_classes": [] + "sha": "7b2d0fa8726b1180c7d6cde4f4afc3800eba7e6f" }, "ElectraForQuestionAnswering": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "ElectraTokenizer", + "ElectraTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ElectraForQuestionAnswering", + "TFElectraForQuestionAnswering" + ], + "sha": "c6b127fd9f3019462e4ca2373762836207e39ce2" }, "ElectraForSequenceClassification": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "ElectraTokenizer", + "ElectraTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ElectraForSequenceClassification", + "TFElectraForSequenceClassification" ], - "processor_classes": [] + "sha": "41f0089ab7876abe0e28dbbd565144acb31f8127" }, "ElectraForTokenClassification": { "tokenizer_classes": [ - "ElectraTokenizerFast", - "ElectraTokenizer" + "ElectraTokenizer", + "ElectraTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ElectraForTokenClassification", + "TFElectraForTokenClassification" + ], + "sha": "1fdbbe70c1ddd16503820a1443d6a379a15ed777" + }, + "ElectraModel": { + "tokenizer_classes": [ + "ElectraTokenizer", + "ElectraTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ElectraModel", + "TFElectraModel" + ], + "sha": "312b532cbef26610d80f2bd008650160cae4f7a1" + }, + "EncoderDecoderModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "EncoderDecoderModel", + "TFEncoderDecoderModel" ], - "processor_classes": [] + "sha": "1038be9fd1b87b2e0a8f33721ff8e4612d34b3b6" }, "ErnieForCausalLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ErnieForCausalLM" + ], + "sha": "b49e00112ff06c2f0a0e54499921dddcf8c3c6a8" }, "ErnieForMaskedLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ErnieForMaskedLM" ], - "processor_classes": [] + "sha": "30429830d1997222d885dcfdbd36d5e02d0d34b1" }, "ErnieForMultipleChoice": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ErnieForMultipleChoice" + ], + "sha": "5a21144bf35dfb60560ff8249116ad4459c0069a" }, "ErnieForNextSentencePrediction": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ErnieForNextSentencePrediction" ], - "processor_classes": [] + "sha": "ed5868efb39bf6afb29f0cf444deafcf1e50b5bc" }, "ErnieForPreTraining": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ErnieForPreTraining" + ], + "sha": "e4ad30d291c310fea25e6f91f91393f993513b42" }, "ErnieForQuestionAnswering": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ErnieForQuestionAnswering" ], - "processor_classes": [] + "sha": "fe7c74b763f63a9fd864dad325385075df7c80c8" }, "ErnieForSequenceClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ErnieForSequenceClassification" + ], + "sha": "84e0be05fcd52f54e96a69f67a2481323a58a9db" }, "ErnieForTokenClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ErnieForTokenClassification" + ], + "sha": "91cf62c43a5a83332552ffa2d8e5e44d63a224ea" + }, + "ErnieMForMultipleChoice": { + "tokenizer_classes": [ + "ErnieMTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "ErnieMForMultipleChoice" + ], + "sha": "c42ee7fcb132a323ace314c32e63c8a7d36ce18f" + }, + "ErnieMForQuestionAnswering": { + "tokenizer_classes": [ + "ErnieMTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "ErnieMForQuestionAnswering" + ], + "sha": "2b90dee75ca87b214f96db00002aa18244ec8e84" + }, + "ErnieMForSequenceClassification": { + "tokenizer_classes": [ + "ErnieMTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "ErnieMForSequenceClassification" + ], + "sha": "d8368646d8b1c67b1460af9c6ec13fd9d894cae6" + }, + "ErnieMForTokenClassification": { + "tokenizer_classes": [ + "ErnieMTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "ErnieMForTokenClassification" + ], + "sha": "a9e29ba60fa0b7bedc2ed26a6b9911427df1ca6b" + }, + "ErnieMModel": { + "tokenizer_classes": [ + "ErnieMTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "ErnieMModel" + ], + "sha": "7306eac3f38c3cf6211f0e741fdb81c6cc92bc09" + }, + "ErnieModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ErnieModel" + ], + "sha": "b51478a9f40e353c41be3a29ccef103dcfe22b4b" }, "EsmForMaskedLM": { "tokenizer_classes": [ "EsmTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "EsmForMaskedLM", + "TFEsmForMaskedLM" + ], + "sha": "b56297b6cd64b9ba7c613d0cd146f1ecbea8115e" }, "EsmForSequenceClassification": { "tokenizer_classes": [ "EsmTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "EsmForSequenceClassification", + "TFEsmForSequenceClassification" + ], + "sha": "cc6d7ef0a4763540d67b7a4fb31bede9a7d3f245" }, "EsmForTokenClassification": { "tokenizer_classes": [ "EsmTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "EsmForTokenClassification", + "TFEsmForTokenClassification" + ], + "sha": "498953f66e260b974c504abbc863ee266d6c84a9" + }, + "EsmModel": { + "tokenizer_classes": [ + "EsmTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "EsmModel", + "TFEsmModel" + ], + "sha": "183838263b70809310117a0761542501acf64c21" }, "FNetForMaskedLM": { "tokenizer_classes": [ "FNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FNetForMaskedLM" + ], + "sha": "235602782fb4b4ff0291c999cc174b4a257f1e7f" }, "FNetForMultipleChoice": { "tokenizer_classes": [ "FNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FNetForMultipleChoice" + ], + "sha": "d84b9ee07323895465a29f234e9109b66fd623cf" }, "FNetForNextSentencePrediction": { "tokenizer_classes": [ "FNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FNetForNextSentencePrediction" + ], + "sha": "9b11a763f599a95c3dff8e4255cd952a04101a65" }, "FNetForPreTraining": { "tokenizer_classes": [ "FNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FNetForPreTraining" + ], + "sha": "ed6ec245f3a8f7b53c7b09b020cfae1f8c4aaf7d" }, "FNetForQuestionAnswering": { "tokenizer_classes": [ "FNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FNetForQuestionAnswering" + ], + "sha": "c77c577acae60cd268b0eebdbffcbd75f8e31141" }, "FNetForSequenceClassification": { "tokenizer_classes": [ "FNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FNetForSequenceClassification" + ], + "sha": "6e6dbab691e5ec18e04b98514f1656dc3a842192" }, "FNetForTokenClassification": { "tokenizer_classes": [ "FNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FNetForTokenClassification" + ], + "sha": "b27e341994ef7913dcdd72326d3475c9668d07d5" + }, + "FNetModel": { + "tokenizer_classes": [ + "FNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "FNetModel" + ], + "sha": "38041395aa488da543dec9ed86318d9ec4d4839e" }, "FSMTForConditionalGeneration": { "tokenizer_classes": [ "FSMTTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FSMTForConditionalGeneration" + ], + "sha": "6a1a981b29c8a98c1fd31bd0ad809f5575ca6c7a" + }, + "FSMTModel": { + "tokenizer_classes": [ + "FSMTTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "FSMTModel" + ], + "sha": "683f6f73a2ab87801f1695a72d1af63cf173ab7c" }, "FlaubertForMultipleChoice": { "tokenizer_classes": [ "FlaubertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FlaubertForMultipleChoice", + "TFFlaubertForMultipleChoice" + ], + "sha": "8b12bd87a63f2e86c3482431742f6d8abf6ec4fd" }, "FlaubertForQuestionAnsweringSimple": { "tokenizer_classes": [ "FlaubertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FlaubertForQuestionAnsweringSimple", + "TFFlaubertForQuestionAnsweringSimple" + ], + "sha": "5c0e7ad1efae7e3497f5cd6d2d9519403df49d37" }, "FlaubertForSequenceClassification": { "tokenizer_classes": [ "FlaubertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FlaubertForSequenceClassification", + "TFFlaubertForSequenceClassification" + ], + "sha": "762f12a8c99690be8ed2663b7af3011660174a7c" }, "FlaubertForTokenClassification": { "tokenizer_classes": [ "FlaubertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FlaubertForTokenClassification", + "TFFlaubertForTokenClassification" + ], + "sha": "d2ab741c937bb69ef27c89e4c86a8c9d444874ca" + }, + "FlaubertModel": { + "tokenizer_classes": [ + "FlaubertTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "FlaubertModel", + "TFFlaubertModel" + ], + "sha": "bdc2f8e17bb869393053429ec8c1c842bfeabb07" + }, + "FlaubertWithLMHeadModel": { + "tokenizer_classes": [ + "FlaubertTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "FlaubertWithLMHeadModel", + "TFFlaubertWithLMHeadModel" + ], + "sha": "f20eb0932c90061003c9cc4e109c6ea22559c4f2" }, "FlavaForPreTraining": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [ + "FlavaImageProcessor" + ], + "model_classes": [ + "FlavaForPreTraining" + ], + "sha": "6e9b2094060a5fa27984c7b49e5d0e820a88b487" + }, + "FlavaModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" ], "processor_classes": [ "FlavaImageProcessor" - ] + ], + "model_classes": [ + "FlavaModel" + ], + "sha": "31ebf1b7a0ef1fd5059b98e28e5ab1c366d2c482" + }, + "FunnelBaseModel": { + "tokenizer_classes": [ + "FunnelTokenizer", + "FunnelTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "FunnelBaseModel", + "TFFunnelBaseModel" + ], + "sha": "87fed4252812df23315a56531625333e315681c6" }, "FunnelForMaskedLM": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "FunnelTokenizer", + "FunnelTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "FunnelForMaskedLM", + "TFFunnelForMaskedLM" ], - "processor_classes": [] + "sha": "5543daf29f185cd45f2599bd6f38c96064c9c8de" }, "FunnelForMultipleChoice": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "FunnelTokenizer", + "FunnelTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FunnelForMultipleChoice", + "TFFunnelForMultipleChoice" + ], + "sha": "a8bf597e37dbefb1ac5c97c4cb162c3d522a33a1" }, "FunnelForPreTraining": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "FunnelTokenizer", + "FunnelTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "FunnelForPreTraining", + "TFFunnelForPreTraining" ], - "processor_classes": [] + "sha": "cbcb300d60aacd5950a45409b6e3f0f240c9082e" }, "FunnelForQuestionAnswering": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "FunnelTokenizer", + "FunnelTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "FunnelForQuestionAnswering", + "TFFunnelForQuestionAnswering" + ], + "sha": "6a5675305e096434e818486a13892cb55daffd13" }, "FunnelForSequenceClassification": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "FunnelTokenizer", + "FunnelTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "FunnelForSequenceClassification", + "TFFunnelForSequenceClassification" ], - "processor_classes": [] + "sha": "1bc557a1e4314da21a44dee57b799e95a7025e5c" }, "FunnelForTokenClassification": { "tokenizer_classes": [ - "FunnelTokenizerFast", - "FunnelTokenizer" + "FunnelTokenizer", + "FunnelTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "FunnelForTokenClassification", + "TFFunnelForTokenClassification" + ], + "sha": "693bc1217a224efd558f410ddc8ffc63739bebc3" + }, + "FunnelModel": { + "tokenizer_classes": [ + "FunnelTokenizer", + "FunnelTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "FunnelModel", + "TFFunnelModel" ], - "processor_classes": [] + "sha": "bfbaa8fa21c3abf80b94e7168b5ecff8ec5b5f76" }, "GLPNForDepthEstimation": { "tokenizer_classes": [], "processor_classes": [ "GLPNImageProcessor" - ] + ], + "model_classes": [ + "GLPNForDepthEstimation" + ], + "sha": "32ca1c1ef5d33242e5e7c0433bcd773c082f0260" + }, + "GLPNModel": { + "tokenizer_classes": [], + "processor_classes": [ + "GLPNImageProcessor" + ], + "model_classes": [ + "GLPNModel" + ], + "sha": "24a8dbb48b1aa0ba2eba44324fcd0c78cca64dd4" }, "GPT2ForSequenceClassification": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPT2ForSequenceClassification", + "TFGPT2ForSequenceClassification" ], - "processor_classes": [] + "sha": "90a2d78e5c7f288152f8456c3d58a43b40a58449" }, "GPT2ForTokenClassification": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPT2ForTokenClassification" + ], + "sha": "da78bc95b45fab2da9d43f2ca27164996e31ade1" + }, + "GPT2LMHeadModel": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPT2LMHeadModel", + "TFGPT2LMHeadModel" + ], + "sha": "78f56535d4ce19e9d7c0992e390085c5a4196b37" + }, + "GPT2Model": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "GPT2Model", + "TFGPT2Model" + ], + "sha": "d6694b0d8fe17978761c9305dc151780506b192e" }, "GPTJForCausalLM": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTJForCausalLM", + "TFGPTJForCausalLM" ], - "processor_classes": [] + "sha": "1fff390baa45cb187903ebdd269c975bb9ed7386" }, "GPTJForQuestionAnswering": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "GPTJForQuestionAnswering", + "TFGPTJForQuestionAnswering" + ], + "sha": "3d4ec61dbed01f844d4c309971eeb5ad722c6c84" }, "GPTJForSequenceClassification": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTJForSequenceClassification", + "TFGPTJForSequenceClassification" + ], + "sha": "4b5db259cd16ca84ae2cd79aa4851cdd14479128" + }, + "GPTJModel": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "GPTJModel", + "TFGPTJModel" + ], + "sha": "d8e1db30d08fbf57da6fc139aea3ffd63ab6226e" }, "GPTNeoForCausalLM": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoForCausalLM" ], - "processor_classes": [] + "sha": "e88934e402c15195dd99b2947632415dd7645268" }, "GPTNeoForSequenceClassification": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoForSequenceClassification" + ], + "sha": "bf2090d5d91a70eb37ba51fbdcf23afc7031fea8" + }, + "GPTNeoModel": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoModel" ], - "processor_classes": [] + "sha": "72a7cd49da613c3125a90884df4763545c594e56" }, "GPTNeoXForCausalLM": { "tokenizer_classes": [ "GPTNeoXTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "GPTNeoXForCausalLM" + ], + "sha": "0229cfaaa843c6b492ac2abffabb00f1ff1936f8" }, "GPTNeoXJapaneseForCausalLM": { "tokenizer_classes": [ "GPTNeoXJapaneseTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "GPTNeoXJapaneseForCausalLM" + ], + "sha": "5fca2479f1064fd22e17f944c8fcc14f7e73f1d5" + }, + "GPTNeoXJapaneseModel": { + "tokenizer_classes": [ + "GPTNeoXJapaneseTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoXJapaneseModel" + ], + "sha": "5c6ed124150df845cfc701d70b97fdcde687be52" + }, + "GPTNeoXModel": { + "tokenizer_classes": [ + "GPTNeoXTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "GPTNeoXModel" + ], + "sha": "33114ba2f72189d5a2bd63f0cdb78551189242ff" + }, + "GPTSanJapaneseForConditionalGeneration": { + "tokenizer_classes": [ + "GPTSanJapaneseTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "GPTSanJapaneseForConditionalGeneration" + ], + "sha": "83bbd0feb62cd12d9163c7638e15bf2bb6fef1eb" + }, + "GitForCausalLM": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [ + "CLIPImageProcessor" + ], + "model_classes": [ + "GitForCausalLM" + ], + "sha": "60f9c50466ae0beeb11776ca5bfeb6473f441554" + }, + "GitModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [ + "CLIPImageProcessor" + ], + "model_classes": [ + "GitModel" + ], + "sha": "3d2eb6bddf95bb4a4e59b045d4e464c730c07f41" + }, + "GroupViTModel": { + "tokenizer_classes": [ + "CLIPTokenizer", + "CLIPTokenizerFast" + ], + "processor_classes": [ + "CLIPImageProcessor" + ], + "model_classes": [ + "GroupViTModel", + "TFGroupViTModel" + ], + "sha": "05a3a02dd46cb9eb078608dec98f633c0cf559ef" }, "HubertForCTC": { "tokenizer_classes": [ @@ -1751,7 +2411,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "HubertForCTC" + ], + "sha": "13431b76106f993eedcff48a75bae590a09b14f7" }, "HubertForSequenceClassification": { "tokenizer_classes": [ @@ -1759,698 +2423,1633 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "HubertForSequenceClassification" + ], + "sha": "d23f46607a900b1a55dfee4b7ed205a6823035b1" + }, + "HubertModel": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "HubertModel", + "TFHubertModel" + ], + "sha": "3224562c86c4669db65ae7defdc5fb555b113e95" }, "IBertForMaskedLM": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "IBertForMaskedLM" ], - "processor_classes": [] + "sha": "e333a9c9d375f4d839b7e9e21d1a1c8dad58d7d1" }, "IBertForMultipleChoice": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "IBertForMultipleChoice" + ], + "sha": "a81f7d64cd7ce5fe6cd726b23d9d14ac5d17bf53" }, "IBertForQuestionAnswering": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "IBertForQuestionAnswering" ], - "processor_classes": [] + "sha": "7b66d13d4d6801a82cbeb7f9fd853ca1630d1f8b" }, "IBertForSequenceClassification": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "IBertForSequenceClassification" + ], + "sha": "309d57145c40f889222fe5df62f14dddf4496b38" + }, + "IBertForTokenClassification": { + "tokenizer_classes": [ + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "IBertForTokenClassification" ], - "processor_classes": [] + "sha": "b032e9bff4b081b78c098b2d8bc610ac035c6ddf" }, - "IBertForTokenClassification": { + "IBertModel": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "IBertModel" + ], + "sha": "6749164c678d4883d455f98b1dfc98c62da8f08b" }, "ImageGPTForCausalImageModeling": { "tokenizer_classes": [], "processor_classes": [ "ImageGPTImageProcessor" - ] + ], + "model_classes": [ + "ImageGPTForCausalImageModeling" + ], + "sha": "9a7d1fc04439ab1d9d690de9c3e7673f08568cdf" }, "ImageGPTForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "ImageGPTImageProcessor" - ] + ], + "model_classes": [ + "ImageGPTForImageClassification" + ], + "sha": "d92c7aed4ba5de74a1f542b736010090e4a58b42" + }, + "ImageGPTModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ImageGPTImageProcessor" + ], + "model_classes": [ + "ImageGPTModel" + ], + "sha": "5a7983e48d5841704733dd0756177680ed50c074" }, "LEDForConditionalGeneration": { "tokenizer_classes": [ - "LEDTokenizerFast", - "LEDTokenizer" + "LEDTokenizer", + "LEDTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LEDForConditionalGeneration", + "TFLEDForConditionalGeneration" + ], + "sha": "a354b49a79351f3ea8ae7776d9f8352ae26cfc14" }, "LEDForQuestionAnswering": { "tokenizer_classes": [ - "LEDTokenizerFast", - "LEDTokenizer" + "LEDTokenizer", + "LEDTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LEDForQuestionAnswering" ], - "processor_classes": [] + "sha": "47c7a75a1e650dae60ff6e9bbab0f2386946670c" }, "LEDForSequenceClassification": { "tokenizer_classes": [ - "LEDTokenizerFast", - "LEDTokenizer" + "LEDTokenizer", + "LEDTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LEDForSequenceClassification" + ], + "sha": "3571e2c9d9f2f2ec0b8fe47090330b128be05126" + }, + "LEDModel": { + "tokenizer_classes": [ + "LEDTokenizer", + "LEDTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LEDModel", + "TFLEDModel" ], - "processor_classes": [] + "sha": "3c3f6eb142545afc570187bfdabfe65d43dafbe4" }, "LayoutLMForMaskedLM": { "tokenizer_classes": [ - "LayoutLMTokenizerFast", - "LayoutLMTokenizer" + "LayoutLMTokenizer", + "LayoutLMTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LayoutLMForMaskedLM", + "TFLayoutLMForMaskedLM" + ], + "sha": "0368bd9bd8fd3eb43b8a3b38962b5345b8765514" }, "LayoutLMForQuestionAnswering": { "tokenizer_classes": [ - "LayoutLMTokenizerFast", - "LayoutLMTokenizer" + "LayoutLMTokenizer", + "LayoutLMTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LayoutLMForQuestionAnswering", + "TFLayoutLMForQuestionAnswering" ], - "processor_classes": [] + "sha": "0d6a4bc614fccfa313c1fb6d132a250929518f85" }, "LayoutLMForSequenceClassification": { "tokenizer_classes": [ - "LayoutLMTokenizerFast", - "LayoutLMTokenizer" + "LayoutLMTokenizer", + "LayoutLMTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LayoutLMForSequenceClassification", + "TFLayoutLMForSequenceClassification" + ], + "sha": "1bd68c73dbf6c8c0526d24fbe2831be82998c440" }, "LayoutLMForTokenClassification": { "tokenizer_classes": [ - "LayoutLMTokenizerFast", - "LayoutLMTokenizer" + "LayoutLMTokenizer", + "LayoutLMTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LayoutLMForTokenClassification", + "TFLayoutLMForTokenClassification" + ], + "sha": "155e7da3f1d786aa39d957b16080c52de4a7efd7" + }, + "LayoutLMModel": { + "tokenizer_classes": [ + "LayoutLMTokenizer", + "LayoutLMTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LayoutLMModel", + "TFLayoutLMModel" + ], + "sha": "14f77b30d267910f11f0fd532a91a6b85ab3a4de" }, "LayoutLMv2ForQuestionAnswering": { "tokenizer_classes": [ - "LayoutLMv2TokenizerFast", - "LayoutLMv2Tokenizer" + "LayoutLMv2Tokenizer", + "LayoutLMv2TokenizerFast" ], "processor_classes": [ "LayoutLMv2ImageProcessor" - ] + ], + "model_classes": [ + "LayoutLMv2ForQuestionAnswering" + ], + "sha": "f452e28dd34d3c38cce046b1cc7b0ada69f587b1" }, "LayoutLMv2ForSequenceClassification": { "tokenizer_classes": [ - "LayoutLMv2TokenizerFast", - "LayoutLMv2Tokenizer" + "LayoutLMv2Tokenizer", + "LayoutLMv2TokenizerFast" ], "processor_classes": [ "LayoutLMv2ImageProcessor" - ] + ], + "model_classes": [ + "LayoutLMv2ForSequenceClassification" + ], + "sha": "b483e08fd143113629ecda3dbfd57e69bfeb5f11" }, "LayoutLMv2ForTokenClassification": { "tokenizer_classes": [ - "LayoutLMv2TokenizerFast", - "LayoutLMv2Tokenizer" + "LayoutLMv2Tokenizer", + "LayoutLMv2TokenizerFast" + ], + "processor_classes": [ + "LayoutLMv2ImageProcessor" + ], + "model_classes": [ + "LayoutLMv2ForTokenClassification" + ], + "sha": "0721ae69bff00ecfff1b3d1521a475cde0253299" + }, + "LayoutLMv2Model": { + "tokenizer_classes": [ + "LayoutLMv2Tokenizer", + "LayoutLMv2TokenizerFast" ], "processor_classes": [ "LayoutLMv2ImageProcessor" - ] + ], + "model_classes": [ + "LayoutLMv2Model" + ], + "sha": "6a1b510769b344979a910a7d0bade613a9ec2dfc" }, "LayoutLMv3ForQuestionAnswering": { "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" ], "processor_classes": [ "LayoutLMv3ImageProcessor" - ] + ], + "model_classes": [ + "LayoutLMv3ForQuestionAnswering", + "TFLayoutLMv3ForQuestionAnswering" + ], + "sha": "4640242388e69cf77ea2dd3ac36ec6f1b26628c8" }, "LayoutLMv3ForSequenceClassification": { "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" ], "processor_classes": [ "LayoutLMv3ImageProcessor" - ] + ], + "model_classes": [ + "LayoutLMv3ForSequenceClassification", + "TFLayoutLMv3ForSequenceClassification" + ], + "sha": "96515f699874cfbfbec7a64c539ae92419e4c6dc" }, "LayoutLMv3ForTokenClassification": { "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" + ], + "processor_classes": [ + "LayoutLMv3ImageProcessor" + ], + "model_classes": [ + "LayoutLMv3ForTokenClassification", + "TFLayoutLMv3ForTokenClassification" + ], + "sha": "ed4ffc464f2028fe50dfc6823f4eda78d34be7e6" + }, + "LayoutLMv3Model": { + "tokenizer_classes": [ + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" ], "processor_classes": [ "LayoutLMv3ImageProcessor" - ] + ], + "model_classes": [ + "LayoutLMv3Model", + "TFLayoutLMv3Model" + ], + "sha": "69725e5e2445e5c1c3aa8a2aa49cfd72e0a44565" }, "LevitForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "LevitImageProcessor" - ] + ], + "model_classes": [ + "LevitForImageClassification" + ], + "sha": "5ae8ccaa1fe1c947cb8ae6499e4a150c668bb9f0" }, "LevitForImageClassificationWithTeacher": { "tokenizer_classes": [], "processor_classes": [ "LevitImageProcessor" - ] + ], + "model_classes": [ + "LevitForImageClassificationWithTeacher" + ], + "sha": "568cc0d965b9bd293f240e7724314db6d50f6722" + }, + "LevitModel": { + "tokenizer_classes": [], + "processor_classes": [ + "LevitImageProcessor" + ], + "model_classes": [ + "LevitModel" + ], + "sha": "172efa52b50c75c3b3e498fa638f55e65b2ebf87" }, "LiltForQuestionAnswering": { "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LiltForQuestionAnswering" + ], + "sha": "0a348441999e98ec003b29fc4d5a67ad22ee6ca2" }, "LiltForSequenceClassification": { "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LiltForSequenceClassification" ], - "processor_classes": [] + "sha": "c53ab0ba33536fe564a4a1e4f1674d990c01b83a" }, "LiltForTokenClassification": { "tokenizer_classes": [ - "LayoutLMv3TokenizerFast", - "LayoutLMv3Tokenizer" + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LiltForTokenClassification" + ], + "sha": "14f85076f9b3f7016917e324d51ebd22511a2ae5" + }, + "LiltModel": { + "tokenizer_classes": [ + "LayoutLMv3Tokenizer", + "LayoutLMv3TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LiltModel" ], - "processor_classes": [] + "sha": "3f1166cc14c532388df7e82336a8e575a813bd3f" }, "LongT5ForConditionalGeneration": { "tokenizer_classes": [ "T5TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LongT5ForConditionalGeneration" + ], + "sha": "3c83684439875f33fc0f79d164e7b446245cbc74" + }, + "LongT5Model": { + "tokenizer_classes": [ + "T5TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LongT5Model" + ], + "sha": "3b825c077d41103476d88bd93a6a9af52194b99e" }, "LongformerForMaskedLM": { "tokenizer_classes": [ - "LongformerTokenizerFast", - "LongformerTokenizer" + "LongformerTokenizer", + "LongformerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LongformerForMaskedLM", + "TFLongformerForMaskedLM" ], - "processor_classes": [] + "sha": "929d3bda9a1485d9bae41f9dbfc1d149c1c4e78e" }, "LongformerForMultipleChoice": { "tokenizer_classes": [ - "LongformerTokenizerFast", - "LongformerTokenizer" + "LongformerTokenizer", + "LongformerTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LongformerForMultipleChoice", + "TFLongformerForMultipleChoice" + ], + "sha": "60b1ecac6b9385ce18c7e6978ab161cce8e7f9d4" }, "LongformerForQuestionAnswering": { "tokenizer_classes": [ - "LongformerTokenizerFast", - "LongformerTokenizer" + "LongformerTokenizer", + "LongformerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LongformerForQuestionAnswering", + "TFLongformerForQuestionAnswering" ], - "processor_classes": [] + "sha": "be45ab1321b703f2200cbbcae560aaf2e2afef88" }, "LongformerForSequenceClassification": { "tokenizer_classes": [ - "LongformerTokenizerFast", - "LongformerTokenizer" + "LongformerTokenizer", + "LongformerTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LongformerForSequenceClassification", + "TFLongformerForSequenceClassification" + ], + "sha": "8bc0de0b0f740bf397eb2770ec3ce3a24f3d7af9" }, "LongformerForTokenClassification": { "tokenizer_classes": [ - "LongformerTokenizerFast", - "LongformerTokenizer" + "LongformerTokenizer", + "LongformerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LongformerForTokenClassification", + "TFLongformerForTokenClassification" + ], + "sha": "efa33a9b6f47f0f7979af08ae8d04a5a7363a14b" + }, + "LongformerModel": { + "tokenizer_classes": [ + "LongformerTokenizer", + "LongformerTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LongformerModel", + "TFLongformerModel" + ], + "sha": "b023d531688e8655fc09300ac36742588efb3240" }, "LukeForMaskedLM": { "tokenizer_classes": [ "LukeTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LukeForMaskedLM" + ], + "sha": "954cf6cd2bf1f298a3956b10c36656c57387506d" }, "LukeForMultipleChoice": { "tokenizer_classes": [ "LukeTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LukeForMultipleChoice" + ], + "sha": "d1310a9174ad50d60b30ad6049e165deb2539034" }, "LukeForQuestionAnswering": { "tokenizer_classes": [ "LukeTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LukeForQuestionAnswering" + ], + "sha": "3ea38da4e32cb4e45bea82b2e81a8639aeba2c35" }, "LukeForSequenceClassification": { "tokenizer_classes": [ "LukeTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LukeForSequenceClassification" + ], + "sha": "b5b11248aeb4f5976379d15a977aeb2677e0c0f9" }, "LukeForTokenClassification": { "tokenizer_classes": [ "LukeTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LukeForTokenClassification" + ], + "sha": "8aab1a33ad26a344a6f4dfd68630e9661e174471" + }, + "LukeModel": { + "tokenizer_classes": [ + "LukeTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "LukeModel" + ], + "sha": "ae23a674e7297d41f33c9af86e039757dfd2d531" }, "LxmertForPreTraining": { "tokenizer_classes": [ - "LxmertTokenizerFast", - "LxmertTokenizer" + "LxmertTokenizer", + "LxmertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LxmertForPreTraining", + "TFLxmertForPreTraining" + ], + "sha": "7b0843403c187aef00f20d5087086468d9613d2c" }, "LxmertForQuestionAnswering": { "tokenizer_classes": [ - "LxmertTokenizerFast", - "LxmertTokenizer" + "LxmertTokenizer", + "LxmertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "LxmertForQuestionAnswering" + ], + "sha": "27a74bd2cd156e46656c43ceb432c4deda0df5c1" + }, + "LxmertModel": { + "tokenizer_classes": [ + "LxmertTokenizer", + "LxmertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "LxmertModel", + "TFLxmertModel" + ], + "sha": "97612a0d6b14406ea9bfd7672e6974e0961cbef1" }, "M2M100ForConditionalGeneration": { "tokenizer_classes": [ "M2M100Tokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "M2M100ForConditionalGeneration" + ], + "sha": "32ac347092d51f658b41ffc111b67d49acdeab46" + }, + "M2M100Model": { + "tokenizer_classes": [ + "M2M100Tokenizer" + ], + "processor_classes": [], + "model_classes": [ + "M2M100Model" + ], + "sha": "e95c2ae168c7ba19f8114def40e1b1edd953b2f5" }, "MBartForCausalLM": { "tokenizer_classes": [ - "MBartTokenizerFast", - "MBartTokenizer" + "MBartTokenizer", + "MBartTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MBartForCausalLM" + ], + "sha": "e0f8bdbcc8afd38ef9a9d78ad96d3e93ddd19d71" }, "MBartForConditionalGeneration": { "tokenizer_classes": [ - "MBartTokenizerFast", - "MBartTokenizer" + "MBartTokenizer", + "MBartTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MBartForConditionalGeneration", + "TFMBartForConditionalGeneration" ], - "processor_classes": [] + "sha": "9d2a48747c3671ef7a9b0c202d79c677b6d6f272" }, "MBartForQuestionAnswering": { "tokenizer_classes": [ - "MBartTokenizerFast", - "MBartTokenizer" + "MBartTokenizer", + "MBartTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MBartForQuestionAnswering" + ], + "sha": "0ad08bf0c1c39d358b92b38f75cbe04609167c3f" }, "MBartForSequenceClassification": { "tokenizer_classes": [ - "MBartTokenizerFast", - "MBartTokenizer" + "MBartTokenizer", + "MBartTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MBartForSequenceClassification" + ], + "sha": "05ed9ea84c02f4a5347f5c235d38f9df891cb811" + }, + "MBartModel": { + "tokenizer_classes": [ + "MBartTokenizer", + "MBartTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MBartModel", + "TFMBartModel" + ], + "sha": "21e9aa6eff4ae280e299ac33c94f1a317300571b" }, "MCTCTForCTC": { "tokenizer_classes": [], "processor_classes": [ "MCTCTFeatureExtractor" - ] + ], + "model_classes": [ + "MCTCTForCTC" + ], + "sha": "895a3d74f87b344b1f0a71eae4f085941d51b5cf" + }, + "MCTCTModel": { + "tokenizer_classes": [], + "processor_classes": [ + "MCTCTFeatureExtractor" + ], + "model_classes": [ + "MCTCTModel" + ], + "sha": "ce73d5c2b6fe163de778697d7b0543bf00d7ffa8" }, "MPNetForMaskedLM": { "tokenizer_classes": [ - "MPNetTokenizerFast", - "MPNetTokenizer" + "MPNetTokenizer", + "MPNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MPNetForMaskedLM", + "TFMPNetForMaskedLM" + ], + "sha": "50af96e7d0202aef86e396c136e4c4fde8afe183" }, "MPNetForMultipleChoice": { "tokenizer_classes": [ - "MPNetTokenizerFast", - "MPNetTokenizer" + "MPNetTokenizer", + "MPNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MPNetForMultipleChoice", + "TFMPNetForMultipleChoice" ], - "processor_classes": [] + "sha": "af4ff8bf296a3a51f5ab6cd9f56741e4c732487c" }, "MPNetForQuestionAnswering": { "tokenizer_classes": [ - "MPNetTokenizerFast", - "MPNetTokenizer" + "MPNetTokenizer", + "MPNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MPNetForQuestionAnswering", + "TFMPNetForQuestionAnswering" + ], + "sha": "3e1a25c0d3243f78f81580c312ada3b39c06b428" }, "MPNetForSequenceClassification": { "tokenizer_classes": [ - "MPNetTokenizerFast", - "MPNetTokenizer" + "MPNetTokenizer", + "MPNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MPNetForSequenceClassification", + "TFMPNetForSequenceClassification" ], - "processor_classes": [] + "sha": "43da45c0a0d73c5a5567b4c7ec512ec5023e52dd" }, "MPNetForTokenClassification": { "tokenizer_classes": [ - "MPNetTokenizerFast", - "MPNetTokenizer" + "MPNetTokenizer", + "MPNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MPNetForTokenClassification", + "TFMPNetForTokenClassification" + ], + "sha": "4e825eff24df533321ebab823eb66ce67e4ab3d9" + }, + "MPNetModel": { + "tokenizer_classes": [ + "MPNetTokenizer", + "MPNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MPNetModel", + "TFMPNetModel" + ], + "sha": "847c68344c2922e9a71fa8835b87a0f6f72b9f47" }, "MarianForCausalLM": { "tokenizer_classes": [ "MarianTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [], + "sha": "5fb205e6db8e18e3c6cdd4e4709be292ba4599f3" + }, + "MarianMTModel": { + "tokenizer_classes": [ + "MarianTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "MarianMTModel", + "TFMarianMTModel" + ], + "sha": "0405f542b31561592231a86e3009d05256cbf49f" + }, + "MarianModel": { + "tokenizer_classes": [ + "MarianTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "MarianModel", + "TFMarianModel" + ], + "sha": "3649748c0286c6d5179a7013a716f7314db182a8" }, "MarkupLMForQuestionAnswering": { "tokenizer_classes": [ - "MarkupLMTokenizerFast", - "MarkupLMTokenizer" + "MarkupLMTokenizer", + "MarkupLMTokenizerFast" ], "processor_classes": [ "MarkupLMFeatureExtractor" - ] + ], + "model_classes": [ + "MarkupLMForQuestionAnswering" + ], + "sha": "c8bb9f93591d980362547b0bdca9f23ace2f383e" }, "MarkupLMForSequenceClassification": { "tokenizer_classes": [ - "MarkupLMTokenizerFast", - "MarkupLMTokenizer" + "MarkupLMTokenizer", + "MarkupLMTokenizerFast" ], "processor_classes": [ "MarkupLMFeatureExtractor" - ] + ], + "model_classes": [ + "MarkupLMForSequenceClassification" + ], + "sha": "c2cb7245d68d76e0a5f993fc8a3de099ecebc68b" }, "MarkupLMForTokenClassification": { "tokenizer_classes": [ - "MarkupLMTokenizerFast", - "MarkupLMTokenizer" + "MarkupLMTokenizer", + "MarkupLMTokenizerFast" + ], + "processor_classes": [ + "MarkupLMFeatureExtractor" + ], + "model_classes": [ + "MarkupLMForTokenClassification" + ], + "sha": "b9f924e82f400de0b34b46ee4ba276d686bd4890" + }, + "MarkupLMModel": { + "tokenizer_classes": [ + "MarkupLMTokenizer", + "MarkupLMTokenizerFast" ], "processor_classes": [ "MarkupLMFeatureExtractor" - ] + ], + "model_classes": [ + "MarkupLMModel" + ], + "sha": "9687ba29f1c59d978e3d4b0fa702031f88eff53b" + }, + "Mask2FormerForUniversalSegmentation": { + "tokenizer_classes": [], + "processor_classes": [ + "Mask2FormerImageProcessor" + ], + "model_classes": [ + "Mask2FormerForUniversalSegmentation" + ], + "sha": "6429a7349527c9ef140ae691b83c47702cce1bc0" + }, + "Mask2FormerModel": { + "tokenizer_classes": [], + "processor_classes": [ + "Mask2FormerImageProcessor" + ], + "model_classes": [ + "Mask2FormerModel" + ], + "sha": "9bee8709204024b3669d503cdfe8890182f2a075" }, "MaskFormerForInstanceSegmentation": { "tokenizer_classes": [], "processor_classes": [ - "MaskFormerFeatureExtractor" - ] + "MaskFormerImageProcessor" + ], + "model_classes": [ + "MaskFormerForInstanceSegmentation" + ], + "sha": "f844aaa81f55cb199c115f1bf95c217a70685570" + }, + "MaskFormerModel": { + "tokenizer_classes": [], + "processor_classes": [ + "MaskFormerImageProcessor" + ], + "model_classes": [ + "MaskFormerModel" + ], + "sha": "473b54a464bc0ccee29bc23b4f6610f32eec05af" }, "MegatronBertForCausalLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MegatronBertForCausalLM" + ], + "sha": "ff08d05ef8f98fdccf1f01560ec6ec4adbc8a3e3" }, "MegatronBertForMaskedLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MegatronBertForMaskedLM" ], - "processor_classes": [] + "sha": "2ed25e2681d26b51b404ef1347a385c5f2c86a9a" }, "MegatronBertForMultipleChoice": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MegatronBertForMultipleChoice" + ], + "sha": "1485af4b75f8f234d2b4b5aea50ab2ec55223a15" }, "MegatronBertForNextSentencePrediction": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MegatronBertForNextSentencePrediction" ], - "processor_classes": [] + "sha": "52bc9ee1d5145344f66b088ed278f07ed3d90584" }, "MegatronBertForPreTraining": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MegatronBertForPreTraining" + ], + "sha": "e580d0efd54e1c92789e39b32929234e36ee427f" }, "MegatronBertForQuestionAnswering": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MegatronBertForQuestionAnswering" ], - "processor_classes": [] + "sha": "7342ba042a3c30c15382d00fcb0521533fc43841" }, "MegatronBertForSequenceClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MegatronBertForSequenceClassification" + ], + "sha": "6a7cd480511d817a1e221c8f7558c55a93baed1b" }, "MegatronBertForTokenClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MegatronBertForTokenClassification" + ], + "sha": "8b5334b6ec5f025293ca861de474b57ca84bc005" + }, + "MegatronBertModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MegatronBertModel" + ], + "sha": "f2457fbe535ba97ea13db049f53618b42e13f047" + }, + "MgpstrForSceneTextRecognition": { + "tokenizer_classes": [], + "processor_classes": [ + "MgpstrProcessor" + ], + "model_classes": [ + "MgpstrForSceneTextRecognition" ], - "processor_classes": [] + "sha": "f197d5bfa1fe27b5f28a6e6d4e3ad229b753450a" }, "MobileBertForMaskedLM": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "MobileBertTokenizer", + "MobileBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MobileBertForMaskedLM", + "TFMobileBertForMaskedLM" + ], + "sha": "d689e737d73ad23aed3aabd3177591fc827d1c62" }, "MobileBertForMultipleChoice": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "MobileBertTokenizer", + "MobileBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MobileBertForMultipleChoice", + "TFMobileBertForMultipleChoice" ], - "processor_classes": [] + "sha": "403d1f88be7eb0c769ff3a8e57eab21cc3e75afb" }, "MobileBertForNextSentencePrediction": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "MobileBertTokenizer", + "MobileBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MobileBertForNextSentencePrediction", + "TFMobileBertForNextSentencePrediction" + ], + "sha": "b4d8836a0f259ee3bca9f230093836c9117c5e4d" }, "MobileBertForPreTraining": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "MobileBertTokenizer", + "MobileBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MobileBertForPreTraining", + "TFMobileBertForPreTraining" ], - "processor_classes": [] + "sha": "fbaa13ea6f9fcebb9fde620dd009d12510440d17" }, "MobileBertForQuestionAnswering": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "MobileBertTokenizer", + "MobileBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MobileBertForQuestionAnswering", + "TFMobileBertForQuestionAnswering" + ], + "sha": "ba6a55cf2daec55bfb220c9bab0bc4ad96510087" }, "MobileBertForSequenceClassification": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "MobileBertTokenizer", + "MobileBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MobileBertForSequenceClassification", + "TFMobileBertForSequenceClassification" ], - "processor_classes": [] + "sha": "17ab35603bec351457e035eef2d0426538071f72" }, "MobileBertForTokenClassification": { "tokenizer_classes": [ - "MobileBertTokenizerFast", - "MobileBertTokenizer" + "MobileBertTokenizer", + "MobileBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MobileBertForTokenClassification", + "TFMobileBertForTokenClassification" + ], + "sha": "dee83e820e6c4f069886a5d1875bf6775897313e" + }, + "MobileBertModel": { + "tokenizer_classes": [ + "MobileBertTokenizer", + "MobileBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MobileBertModel", + "TFMobileBertModel" ], - "processor_classes": [] + "sha": "09b2db33ea798a762eeaf7e727e95f9ea8a6d14f" + }, + "MobileNetV1ForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "MobileNetV1ImageProcessor" + ], + "model_classes": [ + "MobileNetV1ForImageClassification" + ], + "sha": "55023dbd0935f147bf1bccf960cea01ca07e0f0c" + }, + "MobileNetV1Model": { + "tokenizer_classes": [], + "processor_classes": [ + "MobileNetV1ImageProcessor" + ], + "model_classes": [ + "MobileNetV1Model" + ], + "sha": "178bd24528147a028938d6ee5c7e65c969ea37b0" }, "MobileNetV2ForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "MobileNetV2ImageProcessor" - ] + ], + "model_classes": [ + "MobileNetV2ForImageClassification" + ], + "sha": "ff907f740cf9ea91bc3cdf403a94ae28fbb2548a" }, "MobileNetV2ForSemanticSegmentation": { "tokenizer_classes": [], "processor_classes": [ "MobileNetV2ImageProcessor" - ] + ], + "model_classes": [ + "MobileNetV2ForSemanticSegmentation" + ], + "sha": "48adbc340e42882f52b54d4f5dd045e16e9ef2d6" + }, + "MobileNetV2Model": { + "tokenizer_classes": [], + "processor_classes": [ + "MobileNetV2ImageProcessor" + ], + "model_classes": [ + "MobileNetV2Model" + ], + "sha": "e876885828825472a80ef1796d89d60b901813ba" }, "MobileViTForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "MobileViTImageProcessor" - ] + ], + "model_classes": [ + "MobileViTForImageClassification", + "TFMobileViTForImageClassification" + ], + "sha": "7d0b31864f856e00f9e34e8c6781dcc7a8cdaf1e" }, "MobileViTForSemanticSegmentation": { "tokenizer_classes": [], "processor_classes": [ "MobileViTImageProcessor" - ] + ], + "model_classes": [ + "MobileViTForSemanticSegmentation", + "TFMobileViTForSemanticSegmentation" + ], + "sha": "215f727caa3c3fc94fa4df486aa706e5d99d4194" + }, + "MobileViTModel": { + "tokenizer_classes": [], + "processor_classes": [ + "MobileViTImageProcessor" + ], + "model_classes": [ + "MobileViTModel", + "TFMobileViTModel" + ], + "sha": "b3a1452e7cb44b600b21ee14f3d5382366855a46" }, "MvpForCausalLM": { "tokenizer_classes": [ - "MvpTokenizerFast", - "MvpTokenizer" + "MvpTokenizer", + "MvpTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MvpForCausalLM" + ], + "sha": "105e5f2c8a0f20d404cb71795539cda5dd49716d" }, "MvpForConditionalGeneration": { "tokenizer_classes": [ - "MvpTokenizerFast", - "MvpTokenizer" + "MvpTokenizer", + "MvpTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MvpForConditionalGeneration" ], - "processor_classes": [] + "sha": "b0b706f14b2f8aae288cba30ae0064e0be7e888b" }, "MvpForQuestionAnswering": { "tokenizer_classes": [ - "MvpTokenizerFast", - "MvpTokenizer" + "MvpTokenizer", + "MvpTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "MvpForQuestionAnswering" + ], + "sha": "82f152b36a40a4c22edcb146e6eaec636d84fa2d" }, "MvpForSequenceClassification": { "tokenizer_classes": [ - "MvpTokenizerFast", - "MvpTokenizer" + "MvpTokenizer", + "MvpTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MvpForSequenceClassification" + ], + "sha": "506b68544d064001929ee9e6db3752e62972a6aa" + }, + "MvpModel": { + "tokenizer_classes": [ + "MvpTokenizer", + "MvpTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "MvpModel" + ], + "sha": "3f4653184721a2bc029b27706d335ef7ddd219d5" + }, + "NatBackbone": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "NatBackbone" + ], + "sha": "d5cc5eccba4da609c82e9f5c649301b9f9fee9fb" + }, + "NatForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "NatForImageClassification" + ], + "sha": "2ff4c9e73c49c392c02a467e87b5511fd924242a" + }, + "NatModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" ], - "processor_classes": [] + "model_classes": [ + "NatModel" + ], + "sha": "75e9756bb94d0ccdce98a8e963eeecbc66f9d573" }, "NezhaForMaskedLM": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "NezhaForMaskedLM" ], - "processor_classes": [] + "sha": "5991cca4b78f0ed7299259a71f3eeed3f3452b72" }, "NezhaForMultipleChoice": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NezhaForMultipleChoice" + ], + "sha": "0f6e9ec791d85ad4503acdec50b3a120f984016b" }, "NezhaForNextSentencePrediction": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "NezhaForNextSentencePrediction" ], - "processor_classes": [] + "sha": "9a34316c14ec8ecc98ff08e46760915c80098a57" }, "NezhaForPreTraining": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NezhaForPreTraining" + ], + "sha": "6259db427a0073061de352ea819d38a74798edd7" }, "NezhaForQuestionAnswering": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "NezhaForQuestionAnswering" ], - "processor_classes": [] + "sha": "31c6a34e85ae8c41294e0f4ef25044e00e511c4d" }, "NezhaForSequenceClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NezhaForSequenceClassification" + ], + "sha": "db057c308ba2e05f223404de11e1816ce4bd62a9" }, "NezhaForTokenClassification": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "NezhaForTokenClassification" + ], + "sha": "235f4e10b4a59709650c2bece3e342ec153d9cfc" + }, + "NezhaModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NezhaModel" + ], + "sha": "80e05ba7c55bcdd7f4d1387ef9a09a7a8e95b5ac" }, "NystromformerForMaskedLM": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NystromformerForMaskedLM" + ], + "sha": "a21609ab92ea2ea438a4c78f9c2faf0594681fb2" }, "NystromformerForMultipleChoice": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NystromformerForMultipleChoice" + ], + "sha": "417bcc3c7636f1b0a5cbd7371ddce566c302e52d" }, "NystromformerForQuestionAnswering": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NystromformerForQuestionAnswering" + ], + "sha": "206c4a09179f8eda76dab66a0b8d5bd61e031a79" }, "NystromformerForSequenceClassification": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NystromformerForSequenceClassification" + ], + "sha": "0e0f7cc4d7cf9a759ac5d1a2ebbcc6026b4c7df3" }, "NystromformerForTokenClassification": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "NystromformerForTokenClassification" + ], + "sha": "3b140a528dd8d259f4edf229430edc8f7633473e" + }, + "NystromformerModel": { + "tokenizer_classes": [ + "AlbertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "NystromformerModel" + ], + "sha": "761c4db14685ba334ec012a5075847b3439da7d1" }, "OPTForCausalLM": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "OPTForCausalLM", + "TFOPTForCausalLM" + ], + "sha": "190d1f4fc0011d2eaeaa05282e0fbd2445e4b11f" }, "OPTForQuestionAnswering": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "OPTForQuestionAnswering" ], - "processor_classes": [] + "sha": "0fa9277ce10dbc3d0922b354befb684a136af00b" }, "OPTForSequenceClassification": { "tokenizer_classes": [ - "GPT2TokenizerFast", - "GPT2Tokenizer" + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "OPTForSequenceClassification" + ], + "sha": "784ab288ab7280b1853ee400ef10ee2a965df352" + }, + "OPTModel": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "OPTModel", + "TFOPTModel" + ], + "sha": "901d92b8f51edb0ec9614cb185fb66a8b5d364c3" + }, + "OneFormerForUniversalSegmentation": { + "tokenizer_classes": [ + "CLIPTokenizer", + "CLIPTokenizerFast" + ], + "processor_classes": [ + "OneFormerImageProcessor" + ], + "model_classes": [ + "OneFormerForUniversalSegmentation" + ], + "sha": "fee1cfd676acc40f09017702ddac6504f3090d14" + }, + "OneFormerModel": { + "tokenizer_classes": [ + "CLIPTokenizer", + "CLIPTokenizerFast" + ], + "processor_classes": [ + "OneFormerImageProcessor" + ], + "model_classes": [ + "OneFormerModel" ], - "processor_classes": [] + "sha": "4163a79328c78f93ec57942598698a138c19a577" }, "OpenAIGPTForSequenceClassification": { "tokenizer_classes": [ - "OpenAIGPTTokenizerFast", - "OpenAIGPTTokenizer" + "OpenAIGPTTokenizer", + "OpenAIGPTTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "OpenAIGPTForSequenceClassification", + "TFOpenAIGPTForSequenceClassification" + ], + "sha": "c513f7f952935085f7573bf70a1ac3ad8f33434c" + }, + "OpenAIGPTLMHeadModel": { + "tokenizer_classes": [ + "OpenAIGPTTokenizer", + "OpenAIGPTTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "OpenAIGPTLMHeadModel", + "TFOpenAIGPTLMHeadModel" + ], + "sha": "33f59ecd860f7a998483ec7631fe32d257235461" + }, + "OpenAIGPTModel": { + "tokenizer_classes": [ + "OpenAIGPTTokenizer", + "OpenAIGPTTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "OpenAIGPTModel", + "TFOpenAIGPTModel" + ], + "sha": "00f6ec0a3a5276af71d08a26199e0ccbf2556fc9" }, "OwlViTForObjectDetection": { "tokenizer_classes": [ - "CLIPTokenizerFast", - "CLIPTokenizer" + "CLIPTokenizer", + "CLIPTokenizerFast" + ], + "processor_classes": [ + "OwlViTImageProcessor" + ], + "model_classes": [ + "OwlViTForObjectDetection" + ], + "sha": "af958c9164f23d0f12921a8edf687f9aaa6af90e" + }, + "OwlViTModel": { + "tokenizer_classes": [ + "CLIPTokenizer", + "CLIPTokenizerFast" ], "processor_classes": [ - "OwlViTFeatureExtractor" - ] + "OwlViTImageProcessor" + ], + "model_classes": [ + "OwlViTModel" + ], + "sha": "f0e27b2b4e53ba70e05d13dcfea8e85272b292a5" }, "PLBartForCausalLM": { "tokenizer_classes": [ "PLBartTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "PLBartForCausalLM" + ], + "sha": "6ee51133246dbdb18fc3681ebd62d21e421b9bb4" }, "PLBartForConditionalGeneration": { "tokenizer_classes": [ "PLBartTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "PLBartForConditionalGeneration" + ], + "sha": "ba191d28f4678d20b4dfed5fca5944018282cf20" }, "PLBartForSequenceClassification": { "tokenizer_classes": [ "PLBartTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "PLBartForSequenceClassification" + ], + "sha": "02063b3d9707fcff619a4e37a0d6e58f76e39b18" + }, + "PLBartModel": { + "tokenizer_classes": [ + "PLBartTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "PLBartModel" + ], + "sha": "cfbba29169b3f40d800403fc1b53982e1f88c5f8" }, "PegasusForCausalLM": { "tokenizer_classes": [ "PegasusTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "PegasusForCausalLM" + ], + "sha": "bedf3a4a4188e44309417b70243f210c68b0a6f8" }, "PegasusForConditionalGeneration": { "tokenizer_classes": [ "PegasusTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "PegasusForConditionalGeneration", + "TFPegasusForConditionalGeneration" + ], + "sha": "cacda18e0c679447ca1f38895fc08650df13d571" + }, + "PegasusModel": { + "tokenizer_classes": [ + "PegasusTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "PegasusModel", + "TFPegasusModel" + ], + "sha": "a3e9e2ccb99adb9ac068f7ab74946bbdbfed52ec" }, "PegasusXForConditionalGeneration": { "tokenizer_classes": [ "PegasusTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "PegasusXForConditionalGeneration" + ], + "sha": "7edff1791c2c20a6983202836a5b9c732f64631e" + }, + "PegasusXModel": { + "tokenizer_classes": [ + "PegasusTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "PegasusXModel" + ], + "sha": "921751ff0cf59e7cfe909fa6803b5d5ceb0d3059" }, "PerceiverForImageClassificationConvProcessing": { "tokenizer_classes": [ @@ -2458,7 +4057,11 @@ ], "processor_classes": [ "PerceiverImageProcessor" - ] + ], + "model_classes": [ + "PerceiverForImageClassificationConvProcessing" + ], + "sha": "2c1e5e62ebc9d0c931adc8c665fb05bde6c1c1f1" }, "PerceiverForImageClassificationFourier": { "tokenizer_classes": [ @@ -2466,7 +4069,11 @@ ], "processor_classes": [ "PerceiverImageProcessor" - ] + ], + "model_classes": [ + "PerceiverForImageClassificationFourier" + ], + "sha": "88da41b8851b76b8be0dacdb3de023db02bb031a" }, "PerceiverForImageClassificationLearned": { "tokenizer_classes": [ @@ -2474,7 +4081,11 @@ ], "processor_classes": [ "PerceiverImageProcessor" - ] + ], + "model_classes": [ + "PerceiverForImageClassificationLearned" + ], + "sha": "879bd1fa38d3baddb027bb2cacba2d160a741375" }, "PerceiverForMaskedLM": { "tokenizer_classes": [ @@ -2482,7 +4093,11 @@ ], "processor_classes": [ "PerceiverImageProcessor" - ] + ], + "model_classes": [ + "PerceiverForMaskedLM" + ], + "sha": "1d2459cbd281ef72da5682e65102aaca96183045" }, "PerceiverForSequenceClassification": { "tokenizer_classes": [ @@ -2490,220 +4105,589 @@ ], "processor_classes": [ "PerceiverImageProcessor" - ] + ], + "model_classes": [ + "PerceiverForSequenceClassification" + ], + "sha": "576f1f96348f0343458499fbf53d4102b5c0f2ff" + }, + "PerceiverModel": { + "tokenizer_classes": [ + "PerceiverTokenizer" + ], + "processor_classes": [ + "PerceiverImageProcessor" + ], + "model_classes": [ + "PerceiverModel" + ], + "sha": "83ec4d2d61ed62525ee033e13d144817beb29d19" }, "PoolFormerForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "PoolFormerImageProcessor" - ] + ], + "model_classes": [ + "PoolFormerForImageClassification" + ], + "sha": "ef04de5a6896100d457fb9553dd9789c09cca98e" + }, + "PoolFormerModel": { + "tokenizer_classes": [], + "processor_classes": [ + "PoolFormerImageProcessor" + ], + "model_classes": [ + "PoolFormerModel" + ], + "sha": "e8037215ebdbf795329ef6525cdc6aa547f04ace" }, "ProphetNetForCausalLM": { "tokenizer_classes": [ "ProphetNetTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ProphetNetForCausalLM" + ], + "sha": "d40b1e75bbc5ea0839563457aff6eee5bc0bb03e" }, "ProphetNetForConditionalGeneration": { "tokenizer_classes": [ "ProphetNetTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ProphetNetForConditionalGeneration" + ], + "sha": "d842875c41278032af39c03c66902786bb5ff2c7" + }, + "ProphetNetModel": { + "tokenizer_classes": [ + "ProphetNetTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "ProphetNetModel" + ], + "sha": "f1ddbbcc768c7ba54c4d75b319540c1635e65937" }, "ReformerForMaskedLM": { "tokenizer_classes": [ - "ReformerTokenizerFast", - "ReformerTokenizer" + "ReformerTokenizer", + "ReformerTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "ReformerForMaskedLM" + ], + "sha": "1e6431e42c676b525e3215e9e3cc8f1404f9f82b" }, "ReformerForQuestionAnswering": { "tokenizer_classes": [ - "ReformerTokenizerFast", - "ReformerTokenizer" + "ReformerTokenizer", + "ReformerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ReformerForQuestionAnswering" ], - "processor_classes": [] + "sha": "62b43977f244474bd6982c6327d0c57310258fcd" }, "ReformerForSequenceClassification": { "tokenizer_classes": [ - "ReformerTokenizerFast", - "ReformerTokenizer" + "ReformerTokenizer", + "ReformerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ReformerForSequenceClassification" + ], + "sha": "67bd534a990a7dcfa02406987e7f066caa2a30e8" + }, + "ReformerModel": { + "tokenizer_classes": [ + "ReformerTokenizer", + "ReformerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "ReformerModel" + ], + "sha": "a34ddb1389067448e9bc1323de674951cfb4cff1" + }, + "ReformerModelWithLMHead": { + "tokenizer_classes": [ + "ReformerTokenizer", + "ReformerTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [], + "sha": "e7a8addaea8407d4c55e144e48aee04be6cca618" }, "RegNetForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "ConvNextImageProcessor" - ] + ], + "model_classes": [ + "RegNetForImageClassification", + "TFRegNetForImageClassification" + ], + "sha": "5ec67c84fc7944c0c5b386bd26820bc4d1f3b32a" + }, + "RegNetModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ConvNextImageProcessor" + ], + "model_classes": [ + "RegNetModel", + "TFRegNetModel" + ], + "sha": "72375e1401dc8271d4abb6295c9cee376f7b8f1a" }, "RemBertForCausalLM": { "tokenizer_classes": [ "RemBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RemBertForCausalLM", + "TFRemBertForCausalLM" + ], + "sha": "0b85a3f867b04078e2b3070dbb7033475946415f" }, "RemBertForMaskedLM": { "tokenizer_classes": [ "RemBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RemBertForMaskedLM", + "TFRemBertForMaskedLM" + ], + "sha": "706f273263519d12cb35736b3b03a96908f752bf" }, "RemBertForMultipleChoice": { "tokenizer_classes": [ "RemBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RemBertForMultipleChoice", + "TFRemBertForMultipleChoice" + ], + "sha": "e03cd3b6eddb3ad9045e4d6d1053812a393a9417" }, "RemBertForQuestionAnswering": { "tokenizer_classes": [ "RemBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RemBertForQuestionAnswering", + "TFRemBertForQuestionAnswering" + ], + "sha": "882925e63c946dfb9679cd80a4c4909fbf1e4f2c" }, "RemBertForSequenceClassification": { "tokenizer_classes": [ "RemBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RemBertForSequenceClassification", + "TFRemBertForSequenceClassification" + ], + "sha": "d315ecf3be15d52df6d7a137702f896265f112b1" }, "RemBertForTokenClassification": { "tokenizer_classes": [ "RemBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RemBertForTokenClassification", + "TFRemBertForTokenClassification" + ], + "sha": "622e82ab7284c36acb1260a171623b650f0a8f52" + }, + "RemBertModel": { + "tokenizer_classes": [ + "RemBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RemBertModel", + "TFRemBertModel" + ], + "sha": "7dc1b9ee6493a52c377a4ceda0e6a24780c33281" + }, + "ResNetBackbone": { + "tokenizer_classes": [], + "processor_classes": [ + "ConvNextImageProcessor" + ], + "model_classes": [ + "ResNetBackbone" + ], + "sha": "c84a6bcf8af4b6a3403dea3cf4c55965ac39f239" }, "ResNetForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "ConvNextImageProcessor" - ] + ], + "model_classes": [ + "ResNetForImageClassification", + "TFResNetForImageClassification" + ], + "sha": "34a180ad24d80811d420d7aa4fbec4a17751aaf8" + }, + "ResNetModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ConvNextImageProcessor" + ], + "model_classes": [ + "ResNetModel", + "TFResNetModel" + ], + "sha": "fafa6cdf9986c6cfbae360596b3574162430bcd3" }, "RoCBertForCausalLM": { "tokenizer_classes": [ "RoCBertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoCBertForCausalLM" + ], + "sha": "194d8dafc4f4142f8d31e6b4be14b55d812f923b" }, "RoCBertForMaskedLM": { "tokenizer_classes": [ "RoCBertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoCBertForMaskedLM" + ], + "sha": "8bc285f32f3b932dbd56ddf91b1170734d638eeb" }, "RoCBertForMultipleChoice": { "tokenizer_classes": [ "RoCBertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoCBertForMultipleChoice" + ], + "sha": "bb54e5ae021d728022d34b12fee3f087d9486af9" }, "RoCBertForPreTraining": { "tokenizer_classes": [ "RoCBertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoCBertForPreTraining" + ], + "sha": "86ebbd5b0bc84660ad7f505082eff19b86c137c8" }, "RoCBertForQuestionAnswering": { "tokenizer_classes": [ "RoCBertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoCBertForQuestionAnswering" + ], + "sha": "1bfc2dc3d6e76170e6dca1ff32a54a0887ff28a3" }, "RoCBertForSequenceClassification": { "tokenizer_classes": [ "RoCBertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoCBertForSequenceClassification" + ], + "sha": "c329038802241f454273894128fea38b60f7c739" }, "RoCBertForTokenClassification": { "tokenizer_classes": [ "RoCBertTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoCBertForTokenClassification" + ], + "sha": "afe5ec22c2ad1d9ff6e3e64c87eb7555faaa936d" + }, + "RoCBertModel": { + "tokenizer_classes": [ + "RoCBertTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "RoCBertModel" + ], + "sha": "29de5580d5f5d3461a88673e7b4c492a9d8a67a4" }, "RoFormerForCausalLM": { "tokenizer_classes": [ - "RoFormerTokenizerFast", - "RoFormerTokenizer" + "RoFormerTokenizer", + "RoFormerTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RoFormerForCausalLM", + "TFRoFormerForCausalLM" + ], + "sha": "6e074219c6dd8f8b221bbfda64fba100f729f88d" }, "RoFormerForMaskedLM": { "tokenizer_classes": [ - "RoFormerTokenizerFast", - "RoFormerTokenizer" + "RoFormerTokenizer", + "RoFormerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RoFormerForMaskedLM", + "TFRoFormerForMaskedLM" ], - "processor_classes": [] + "sha": "a3a4d05f9b29601553a77244f2adcf8194f9367c" }, "RoFormerForMultipleChoice": { "tokenizer_classes": [ - "RoFormerTokenizerFast", - "RoFormerTokenizer" + "RoFormerTokenizer", + "RoFormerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RoFormerForMultipleChoice", + "TFRoFormerForMultipleChoice" + ], + "sha": "aca3999a1d14f09644faed44e2cdfb28ed68a3d3" + }, + "RoFormerForQuestionAnswering": { + "tokenizer_classes": [ + "RoFormerTokenizer", + "RoFormerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RoFormerForQuestionAnswering", + "TFRoFormerForQuestionAnswering" + ], + "sha": "b8a20b3a788f178b9ef64e2eb9587f693dca1b69" + }, + "RoFormerForSequenceClassification": { + "tokenizer_classes": [ + "RoFormerTokenizer", + "RoFormerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RoFormerForSequenceClassification", + "TFRoFormerForSequenceClassification" + ], + "sha": "d092e2d5e62012bf4ec921e763b37865d6189216" + }, + "RoFormerForTokenClassification": { + "tokenizer_classes": [ + "RoFormerTokenizer", + "RoFormerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RoFormerForTokenClassification", + "TFRoFormerForTokenClassification" + ], + "sha": "85d3a17062e1f3e0539abfe738a88203e25349b6" + }, + "RoFormerModel": { + "tokenizer_classes": [ + "RoFormerTokenizer", + "RoFormerTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RoFormerModel", + "TFRoFormerModel" + ], + "sha": "22e7df2f4cd66caf449f2342f63d176005afccc9" + }, + "RobertaForCausalLM": { + "tokenizer_classes": [ + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaForCausalLM", + "TFRobertaForCausalLM" + ], + "sha": "5d1d24d56f9735402e50a2ea513ffde44487733e" + }, + "RobertaForMaskedLM": { + "tokenizer_classes": [ + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaForMaskedLM", + "TFRobertaForMaskedLM" + ], + "sha": "b21c9daf0b3b66530bf5d45d67df5ec392b5059c" + }, + "RobertaForMultipleChoice": { + "tokenizer_classes": [ + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaForMultipleChoice", + "TFRobertaForMultipleChoice" + ], + "sha": "10020d9546d4d7318f4d514fe13daaad07e6269f" + }, + "RobertaForQuestionAnswering": { + "tokenizer_classes": [ + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaForQuestionAnswering", + "TFRobertaForQuestionAnswering" + ], + "sha": "eea4a81306891746bac9e7715f805a2d9dbf4be7" + }, + "RobertaForSequenceClassification": { + "tokenizer_classes": [ + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaForSequenceClassification", + "TFRobertaForSequenceClassification" ], - "processor_classes": [] + "sha": "6a6f53fc6ab98e29ed539e76b1cb76d25a2cd720" }, - "RoFormerForQuestionAnswering": { + "RobertaForTokenClassification": { "tokenizer_classes": [ - "RoFormerTokenizerFast", - "RoFormerTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "RobertaForTokenClassification", + "TFRobertaForTokenClassification" + ], + "sha": "9190044c4091eb0d98ae7638c453e24846bca5d7" }, - "RoFormerForSequenceClassification": { + "RobertaModel": { "tokenizer_classes": [ - "RoFormerTokenizerFast", - "RoFormerTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaModel", + "TFRobertaModel" ], - "processor_classes": [] + "sha": "181a0b8a7ad24500ec327ad07ddb225f0680ac0a" }, - "RoFormerForTokenClassification": { + "RobertaPreLayerNormForCausalLM": { "tokenizer_classes": [ - "RoFormerTokenizerFast", - "RoFormerTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaPreLayerNormForCausalLM", + "TFRobertaPreLayerNormForCausalLM" ], - "processor_classes": [] + "sha": "73b6d4531b41f295a5d310d7aa44736004a59865" }, - "RobertaForCausalLM": { + "RobertaPreLayerNormForMaskedLM": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaPreLayerNormForMaskedLM", + "TFRobertaPreLayerNormForMaskedLM" ], - "processor_classes": [] + "sha": "a61723c77e5ab7adc95285e7823a0a49b99af395" }, - "RobertaForMaskedLM": { + "RobertaPreLayerNormForMultipleChoice": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaPreLayerNormForMultipleChoice", + "TFRobertaPreLayerNormForMultipleChoice" ], - "processor_classes": [] + "sha": "3dcfa62e0771358c60232a18135bfe7c7f6d715e" }, - "RobertaForMultipleChoice": { + "RobertaPreLayerNormForQuestionAnswering": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaPreLayerNormForQuestionAnswering", + "TFRobertaPreLayerNormForQuestionAnswering" ], - "processor_classes": [] + "sha": "a8e76a5a50f7df60055e5ed6a1c3af2e7d34cf01" }, - "RobertaForQuestionAnswering": { + "RobertaPreLayerNormForSequenceClassification": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaPreLayerNormForSequenceClassification", + "TFRobertaPreLayerNormForSequenceClassification" ], - "processor_classes": [] + "sha": "7509cb0286d146ef2fc6beb8867ae31b92fb1b16" }, - "RobertaForSequenceClassification": { + "RobertaPreLayerNormForTokenClassification": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaPreLayerNormForTokenClassification", + "TFRobertaPreLayerNormForTokenClassification" ], - "processor_classes": [] + "sha": "3ad5814ba126b41e18c1978c970e396fab6da9bf" }, - "RobertaForTokenClassification": { + "RobertaPreLayerNormModel": { "tokenizer_classes": [ - "RobertaTokenizerFast", - "RobertaTokenizer" + "RobertaTokenizer", + "RobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "RobertaPreLayerNormModel", + "TFRobertaPreLayerNormModel" ], - "processor_classes": [] + "sha": "4830db38fd310404c5ab70bd00684eca0bc06ca8" }, "SEWDForCTC": { "tokenizer_classes": [ @@ -2711,7 +4695,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "SEWDForCTC" + ], + "sha": "5c7495c77ae9e0f12c0de05d3a5fb95bdcd91768" }, "SEWDForSequenceClassification": { "tokenizer_classes": [ @@ -2719,7 +4707,23 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "SEWDForSequenceClassification" + ], + "sha": "d6cbf1164ce1999fdaf3deeb7a6eba19a3b1f873" + }, + "SEWDModel": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "SEWDModel" + ], + "sha": "dde4e02219449f149bb3403bbeae127cafaf9c79" }, "SEWForCTC": { "tokenizer_classes": [ @@ -2727,7 +4731,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "SEWForCTC" + ], + "sha": "4477c7a277059fba08772acf91cf3e3dd3cb073b" }, "SEWForSequenceClassification": { "tokenizer_classes": [ @@ -2735,19 +4743,56 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "SEWForSequenceClassification" + ], + "sha": "3b90fbb1c0c3848fed18f91a0169bb297a3e6619" + }, + "SEWModel": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "SEWModel" + ], + "sha": "0a0fbb844eeefa0dce62bd05db30a2bb91e5dc88" }, "SegformerForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "SegformerImageProcessor" - ] + ], + "model_classes": [ + "SegformerForImageClassification", + "TFSegformerForImageClassification" + ], + "sha": "c566ae0ed382be4ed61ed6dacffa2ba663e9cc19" }, "SegformerForSemanticSegmentation": { "tokenizer_classes": [], "processor_classes": [ "SegformerImageProcessor" - ] + ], + "model_classes": [ + "SegformerForSemanticSegmentation", + "TFSegformerForSemanticSegmentation" + ], + "sha": "b73798972cdf24daafa858994713aca60e2bf90d" + }, + "SegformerModel": { + "tokenizer_classes": [], + "processor_classes": [ + "SegformerImageProcessor" + ], + "model_classes": [ + "SegformerModel", + "TFSegformerModel" + ], + "sha": "3d4ba8ed2bdf801e6afa855b9d77893f2b7f9e10" }, "Speech2TextForConditionalGeneration": { "tokenizer_classes": [ @@ -2755,116 +4800,420 @@ ], "processor_classes": [ "Speech2TextFeatureExtractor" - ] + ], + "model_classes": [ + "Speech2TextForConditionalGeneration", + "TFSpeech2TextForConditionalGeneration" + ], + "sha": "1da80293ec78762e136cf6dd64b652693f9ab364" + }, + "Speech2TextModel": { + "tokenizer_classes": [ + "Speech2TextTokenizer" + ], + "processor_classes": [ + "Speech2TextFeatureExtractor" + ], + "model_classes": [ + "Speech2TextModel", + "TFSpeech2TextModel" + ], + "sha": "7c6e63bd0c15dd99ef01573d4c43f90e4920cc91" + }, + "SpeechEncoderDecoderModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "SpeechEncoderDecoderModel" + ], + "sha": "78602ae0857728e95de4042bdca8a31ef818890a" + }, + "SpeechT5ForSpeechToText": { + "tokenizer_classes": [ + "SpeechT5Tokenizer" + ], + "processor_classes": [ + "SpeechT5FeatureExtractor" + ], + "model_classes": [ + "SpeechT5ForSpeechToText" + ], + "sha": "d46f0a83324e5865420a27a738ef203292de3479" + }, + "SpeechT5Model": { + "tokenizer_classes": [ + "SpeechT5Tokenizer" + ], + "processor_classes": [ + "SpeechT5FeatureExtractor" + ], + "model_classes": [ + "SpeechT5Model" + ], + "sha": "7b248f77ca88ffddcdb538e772f6de63a86a4f9b" }, "SplinterForPreTraining": { - "tokenizer_classes": [], - "processor_classes": [] + "tokenizer_classes": [ + "SplinterTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "SplinterForPreTraining" + ], + "sha": "e8a94efa740f1d685fa553f49132c6f022de5389" }, "SplinterForQuestionAnswering": { - "tokenizer_classes": [], - "processor_classes": [] + "tokenizer_classes": [ + "SplinterTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "SplinterForQuestionAnswering" + ], + "sha": "d038b7b683face4a361ab0f474d8a5b111c44c4d" + }, + "SplinterModel": { + "tokenizer_classes": [ + "SplinterTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "SplinterModel" + ], + "sha": "a35b13cbb7faba46dc265761bb839267eb53d248" }, "SqueezeBertForMaskedLM": { "tokenizer_classes": [ - "SqueezeBertTokenizerFast", - "SqueezeBertTokenizer" + "SqueezeBertTokenizer", + "SqueezeBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "SqueezeBertForMaskedLM" + ], + "sha": "33ce239408c22d2c98be63c9ab4607ef9ceb6d49" }, "SqueezeBertForMultipleChoice": { "tokenizer_classes": [ - "SqueezeBertTokenizerFast", - "SqueezeBertTokenizer" + "SqueezeBertTokenizer", + "SqueezeBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "SqueezeBertForMultipleChoice" ], - "processor_classes": [] + "sha": "7e9e666896420c7839e27dcb280981d034ba4da5" }, "SqueezeBertForQuestionAnswering": { "tokenizer_classes": [ - "SqueezeBertTokenizerFast", - "SqueezeBertTokenizer" + "SqueezeBertTokenizer", + "SqueezeBertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "SqueezeBertForQuestionAnswering" + ], + "sha": "bceb045a9ac6eb2ded7d358ed577c6dc28ea487a" }, "SqueezeBertForSequenceClassification": { "tokenizer_classes": [ - "SqueezeBertTokenizerFast", - "SqueezeBertTokenizer" + "SqueezeBertTokenizer", + "SqueezeBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "SqueezeBertForSequenceClassification" ], - "processor_classes": [] + "sha": "c5aeb1f454a1d059d41a5f8dacaf784b9de0b899" }, "SqueezeBertForTokenClassification": { "tokenizer_classes": [ - "SqueezeBertTokenizerFast", - "SqueezeBertTokenizer" + "SqueezeBertTokenizer", + "SqueezeBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "SqueezeBertForTokenClassification" + ], + "sha": "70ba60ca44a380e6aa983a37b163c57217219df7" + }, + "SqueezeBertModel": { + "tokenizer_classes": [ + "SqueezeBertTokenizer", + "SqueezeBertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "SqueezeBertModel" ], - "processor_classes": [] + "sha": "e0a3ac56a4047da3f921638252ead5e44438bbdb" + }, + "Swin2SRModel": { + "tokenizer_classes": [], + "processor_classes": [ + "Swin2SRImageProcessor" + ], + "model_classes": [ + "Swin2SRModel" + ], + "sha": "c67f6ecff9ef8675c3869c987277b0a1e040f4be" + }, + "SwinBackbone": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "SwinBackbone" + ], + "sha": "89b28b8ec05a7b3357be75a77eb7809e6fd5cfef" }, "SwinForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "SwinForImageClassification", + "TFSwinForImageClassification" + ], + "sha": "e3c2e80f380ef79781313981da1a993dd8b8d34d" }, "SwinForMaskedImageModeling": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "SwinForMaskedImageModeling", + "TFSwinForMaskedImageModeling" + ], + "sha": "d84b061fbace1bc6e697e3253e222de42053f978" + }, + "SwinModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "SwinModel", + "TFSwinModel" + ], + "sha": "23ff641295660ec4fea399be8aa1bc14565961f8" }, "Swinv2ForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "Swinv2ForImageClassification" + ], + "sha": "3fd755cdf4cf611db83f72f9c9b00eb9257a38ca" }, "Swinv2ForMaskedImageModeling": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "Swinv2ForMaskedImageModeling" + ], + "sha": "8375c31eb6231fde36ec6533a34ba5b28e296163" + }, + "Swinv2Model": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "Swinv2Model" + ], + "sha": "70aeb72e8a266f668c8b51a517ec01003b8d6804" }, "SwitchTransformersForConditionalGeneration": { "tokenizer_classes": [ "T5TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "SwitchTransformersForConditionalGeneration" + ], + "sha": "332cf96ed681e6cadb5b0788d1925224b62f9cf4" + }, + "SwitchTransformersModel": { + "tokenizer_classes": [ + "T5TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "SwitchTransformersModel" + ], + "sha": "a367b372751ab0e647724c25fc3698f5d46bce20" }, "T5ForConditionalGeneration": { "tokenizer_classes": [ "T5TokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "T5ForConditionalGeneration", + "TFT5ForConditionalGeneration" + ], + "sha": "718845103ac90ab0c0594b6a7149ca10e5680f7e" + }, + "T5Model": { + "tokenizer_classes": [ + "T5TokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "T5Model", + "TFT5Model" + ], + "sha": "e47412452747fd8a9921371fa8f7cc92c9be973f" }, "TableTransformerForObjectDetection": { "tokenizer_classes": [], "processor_classes": [ - "DetrFeatureExtractor" - ] + "DetrImageProcessor" + ], + "model_classes": [ + "TableTransformerForObjectDetection" + ], + "sha": "9cf1e3f5c3555a727672a32b49f8b96c5aa20be6" + }, + "TableTransformerModel": { + "tokenizer_classes": [], + "processor_classes": [ + "DetrImageProcessor" + ], + "model_classes": [ + "TableTransformerModel" + ], + "sha": "7b446244d8739b0c29d98f7d537b15ad578577d5" }, "TapasForMaskedLM": { "tokenizer_classes": [ "TapasTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFTapasForMaskedLM", + "TapasForMaskedLM" + ], + "sha": "2cedb92dd9a3dc37ffb7d35ad5190b110992577c" }, "TapasForQuestionAnswering": { "tokenizer_classes": [ "TapasTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFTapasForQuestionAnswering", + "TapasForQuestionAnswering" + ], + "sha": "4cc91b9e5db662e6e392d8052587ae419896d72b" }, "TapasForSequenceClassification": { "tokenizer_classes": [ "TapasTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFTapasForSequenceClassification", + "TapasForSequenceClassification" + ], + "sha": "7c37bfb87a6fce2f8604bb3cab2a14e09a285e14" + }, + "TapasModel": { + "tokenizer_classes": [ + "TapasTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "TFTapasModel", + "TapasModel" + ], + "sha": "bc004af0a415afe1f566c3afe8dd4d48d08c1ce0" + }, + "TimesformerForVideoClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "VideoMAEImageProcessor" + ], + "model_classes": [ + "TimesformerForVideoClassification" + ], + "sha": "0b3b8e314618d7af34fb44477745491b44bf556d" + }, + "TimesformerModel": { + "tokenizer_classes": [], + "processor_classes": [ + "VideoMAEImageProcessor" + ], + "model_classes": [ + "TimesformerModel" + ], + "sha": "ea51f7ebb6426ad2b1fa1396e83f8e8ad5bc3b44" }, "TransfoXLForSequenceClassification": { "tokenizer_classes": [ "TransfoXLTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFTransfoXLForSequenceClassification", + "TransfoXLForSequenceClassification" + ], + "sha": "f3d370184350667d74056b979081b0bf5b0083c1" + }, + "TransfoXLLMHeadModel": { + "tokenizer_classes": [ + "TransfoXLTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "TFTransfoXLLMHeadModel", + "TransfoXLLMHeadModel" + ], + "sha": "e0d4cebcdde52d8d4c81782a1edc606830bd6afd" + }, + "TransfoXLModel": { + "tokenizer_classes": [ + "TransfoXLTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "TFTransfoXLModel", + "TransfoXLModel" + ], + "sha": "6938eeae35662a862accb01412dfc486454bdc8f" + }, + "TvltForPreTraining": { + "tokenizer_classes": [], + "processor_classes": [ + "TvltProcessor" + ], + "model_classes": [ + "TvltForPreTraining" + ], + "sha": "f7bd2833764eb6d55a921aaed81d3f21119016ae" + }, + "TvltModel": { + "tokenizer_classes": [], + "processor_classes": [ + "TvltProcessor" + ], + "model_classes": [ + "TvltModel" + ], + "sha": "c3cbf7a6159c038f333ce7adda2480ea3396b2b3" }, "UniSpeechForCTC": { "tokenizer_classes": [ @@ -2872,7 +5221,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "UniSpeechForCTC" + ], + "sha": "102b56d76f4d74cface309801c0ad80892583751" }, "UniSpeechForPreTraining": { "tokenizer_classes": [ @@ -2880,7 +5233,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "UniSpeechForPreTraining" + ], + "sha": "830be5b3e85aaae7bcc961218e417c29743d6042" }, "UniSpeechForSequenceClassification": { "tokenizer_classes": [ @@ -2888,7 +5245,23 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "UniSpeechForSequenceClassification" + ], + "sha": "a30ac1516944757ccd8efcbcf94033a03f8708bf" + }, + "UniSpeechModel": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "UniSpeechModel" + ], + "sha": "18e170eb1091715b74ace28c8c380b6bf2b6202d" }, "UniSpeechSatForCTC": { "tokenizer_classes": [ @@ -2896,7 +5269,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "UniSpeechSatForCTC" + ], + "sha": "a8617538d3a2ae990f022bb0c36b8428a4870822" }, "UniSpeechSatForPreTraining": { "tokenizer_classes": [ @@ -2904,7 +5281,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "UniSpeechSatForPreTraining" + ], + "sha": "a772f66db0ab49e1050e524d7fcbe5106ebdaf96" }, "UniSpeechSatForSequenceClassification": { "tokenizer_classes": [ @@ -2912,7 +5293,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "UniSpeechSatForSequenceClassification" + ], + "sha": "f1c16567bd829a6d8a7a2d167d22e9653149e625" }, "UniSpeechSatForXVector": { "tokenizer_classes": [ @@ -2920,65 +5305,252 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "UniSpeechSatForXVector" + ], + "sha": "71cb3780cf3678f74fba00e19df82df76dca6133" + }, + "UniSpeechSatModel": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "UniSpeechSatModel" + ], + "sha": "ea755bbc7c6c6aa649c58b4b000f243acbbd6b5a" + }, + "UperNetForSemanticSegmentation": { + "tokenizer_classes": [], + "processor_classes": [ + "SegformerImageProcessor" + ], + "model_classes": [ + "UperNetForSemanticSegmentation" + ], + "sha": "f1871cb388bc0b203f5397bfc06a373736c2fb9c" + }, + "VanForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "ConvNextImageProcessor" + ], + "model_classes": [ + "VanForImageClassification" + ], + "sha": "694eb147bc4768aeabeffbfb97732281b71a621d" }, - "VanForImageClassification": { + "VanModel": { "tokenizer_classes": [], "processor_classes": [ "ConvNextImageProcessor" - ] + ], + "model_classes": [ + "VanModel" + ], + "sha": "d8ac60ce952020f2b0355fc566d634b2c5ba635d" }, "ViTForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "TFViTForImageClassification", + "ViTForImageClassification" + ], + "sha": "5b3b44a3ed492070c273e481e30ecf4deddc5ec3" }, "ViTForMaskedImageModeling": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "ViTForMaskedImageModeling" + ], + "sha": "d984e0b432fe195c2c26952d4f249031e7b1e2ea" + }, + "ViTHybridForImageClassification": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTHybridImageProcessor" + ], + "model_classes": [ + "ViTHybridForImageClassification" + ], + "sha": "69c7c396032ffe60d54953b584394899fb95ccc1" + }, + "ViTHybridModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTHybridImageProcessor" + ], + "model_classes": [ + "ViTHybridModel" + ], + "sha": "077443bfefe40d625314dbd274d2ff8089624797" }, "ViTMAEForPreTraining": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "TFViTMAEForPreTraining", + "ViTMAEForPreTraining" + ], + "sha": "2d98d80d9c45eef0d5b6f5426d7196bb546fe9fc" + }, + "ViTMAEModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "TFViTMAEModel", + "ViTMAEModel" + ], + "sha": "c7c2f12c19d2dbec08851a9dac7485909629a5fd" }, "ViTMSNForImageClassification": { "tokenizer_classes": [], "processor_classes": [ "ViTImageProcessor" - ] + ], + "model_classes": [ + "ViTMSNForImageClassification" + ], + "sha": "feda819aa7dbb55d850130f4cf1d210858d7eb89" + }, + "ViTMSNModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "ViTMSNModel" + ], + "sha": "0733abf168cb47a149821fdd2113d546e15c47de" + }, + "ViTModel": { + "tokenizer_classes": [], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "TFViTModel", + "ViTModel" + ], + "sha": "31817b7a64ebc3333fcd4801dfbb356ab07b13dd" }, "VideoMAEForPreTraining": { "tokenizer_classes": [], "processor_classes": [ "VideoMAEImageProcessor" - ] + ], + "model_classes": [ + "VideoMAEForPreTraining" + ], + "sha": "9de66c4bb759dc7269a7af17bf70b3194550acaa" }, "VideoMAEForVideoClassification": { "tokenizer_classes": [], "processor_classes": [ "VideoMAEImageProcessor" - ] + ], + "model_classes": [ + "VideoMAEForVideoClassification" + ], + "sha": "d3f743408386bc0ffe2d979de35335e87bc34aec" + }, + "VideoMAEModel": { + "tokenizer_classes": [], + "processor_classes": [ + "VideoMAEImageProcessor" + ], + "model_classes": [ + "VideoMAEModel" + ], + "sha": "a2be96beba888817d92b67525601569d830342ff" }, "ViltForQuestionAnswering": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [ + "ViltImageProcessor" + ], + "model_classes": [ + "ViltForQuestionAnswering" + ], + "sha": "faeffbf43da6621717d8b13e7ebe87d58d750cb2" + }, + "ViltModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" ], "processor_classes": [ "ViltImageProcessor" - ] + ], + "model_classes": [ + "ViltModel" + ], + "sha": "3a89b7b5782947c4f4125162ffe1c9cc18c9c800" + }, + "VisionEncoderDecoderModel": { + "tokenizer_classes": [ + "GPT2Tokenizer", + "GPT2TokenizerFast" + ], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "TFVisionEncoderDecoderModel", + "VisionEncoderDecoderModel" + ], + "sha": "23917761070cf16b26a6d033b6bff9100bbc618b" + }, + "VisionTextDualEncoderModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [ + "ViTImageProcessor" + ], + "model_classes": [ + "VisionTextDualEncoderModel" + ], + "sha": "fcedabfb7cbe3c717c1485613064418acf57ab3d" }, "VisualBertForPreTraining": { "tokenizer_classes": [ - "BertTokenizerFast", - "BertTokenizer" + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "VisualBertForPreTraining" + ], + "sha": "ce5a4d93ce762971cd216cda9aef8b9ce3f0450b" + }, + "VisualBertModel": { + "tokenizer_classes": [ + "BertTokenizer", + "BertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "VisualBertModel" ], - "processor_classes": [] + "sha": "85020189fb7bf1217eb9370b09bca8ec5bcfdafa" }, "Wav2Vec2ConformerForCTC": { "tokenizer_classes": [ @@ -2986,7 +5558,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ConformerForCTC" + ], + "sha": "a2ecb2985fcbb9f3ed000c12c1af6da36f5eaa3a" }, "Wav2Vec2ConformerForPreTraining": { "tokenizer_classes": [ @@ -2994,7 +5570,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ConformerForPreTraining" + ], + "sha": "099279b69e5da19efb05589804ccee210a0e57ae" }, "Wav2Vec2ConformerForSequenceClassification": { "tokenizer_classes": [ @@ -3002,7 +5582,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ConformerForSequenceClassification" + ], + "sha": "e8c1bca543c54bf15a6c026cb3761993b52cf617" }, "Wav2Vec2ConformerForXVector": { "tokenizer_classes": [ @@ -3010,7 +5594,23 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ConformerForXVector" + ], + "sha": "ba206a55998f16e134960728bd02006eaf39114f" + }, + "Wav2Vec2ConformerModel": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "Wav2Vec2ConformerModel" + ], + "sha": "ef2fe3aa8c23e6f8696e6612061aaddecae49994" }, "Wav2Vec2ForCTC": { "tokenizer_classes": [ @@ -3018,7 +5618,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ForCTC" + ], + "sha": "6245fbb1cb99cea5c4de1e73f81fba978fb275ac" }, "Wav2Vec2ForMaskedLM": { "tokenizer_classes": [ @@ -3026,7 +5630,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ForMaskedLM" + ], + "sha": "e083cf4fefec4df3c241dbbe5e17a84a794a89bd" }, "Wav2Vec2ForPreTraining": { "tokenizer_classes": [ @@ -3034,7 +5642,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ForPreTraining" + ], + "sha": "a8d71e216334260353ccbf5ce84cd6924f7457da" }, "Wav2Vec2ForSequenceClassification": { "tokenizer_classes": [ @@ -3042,7 +5654,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ForSequenceClassification" + ], + "sha": "2000b2022abcc37100241485f5872126b70164c9" }, "Wav2Vec2ForXVector": { "tokenizer_classes": [ @@ -3050,7 +5666,24 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "Wav2Vec2ForXVector" + ], + "sha": "f4c422db53aae061ea609f4407af7cd5b33c8942" + }, + "Wav2Vec2Model": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "TFWav2Vec2Model", + "Wav2Vec2Model" + ], + "sha": "7a998ee3ee0619a52828a79c3eed6872fd053f37" }, "WavLMForCTC": { "tokenizer_classes": [ @@ -3058,7 +5691,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "WavLMForCTC" + ], + "sha": "f1139c5ddf34d2327ae1f6917edd7da180b06971" }, "WavLMForSequenceClassification": { "tokenizer_classes": [ @@ -3066,7 +5703,11 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "WavLMForSequenceClassification" + ], + "sha": "4ba5f2019b46866ce2011c993194ebda60afc028" }, "WavLMForXVector": { "tokenizer_classes": [ @@ -3074,144 +5715,443 @@ ], "processor_classes": [ "Wav2Vec2FeatureExtractor" - ] + ], + "model_classes": [ + "WavLMForXVector" + ], + "sha": "faf9264eac56a56d5510a0984d7e1146e4c8cf62" + }, + "WavLMModel": { + "tokenizer_classes": [ + "Wav2Vec2CTCTokenizer" + ], + "processor_classes": [ + "Wav2Vec2FeatureExtractor" + ], + "model_classes": [ + "WavLMModel" + ], + "sha": "e932275e37cb643be271f655bd1d649f4f4b4bd5" }, "WhisperForConditionalGeneration": { "tokenizer_classes": [ - "WhisperTokenizer" + "WhisperTokenizer", + "WhisperTokenizerFast" + ], + "processor_classes": [ + "WhisperFeatureExtractor" + ], + "model_classes": [ + "TFWhisperForConditionalGeneration", + "WhisperForConditionalGeneration" + ], + "sha": "598101b885b24508042d9292e54aa04bff96318e" + }, + "WhisperModel": { + "tokenizer_classes": [ + "WhisperTokenizer", + "WhisperTokenizerFast" ], "processor_classes": [ "WhisperFeatureExtractor" - ] + ], + "model_classes": [ + "TFWhisperModel", + "WhisperModel" + ], + "sha": "c04c50216bb6b0a8f4d55f2fa9f9f4cf61c8a77c" + }, + "XCLIPModel": { + "tokenizer_classes": [ + "CLIPTokenizer", + "CLIPTokenizerFast" + ], + "processor_classes": [ + "VideoMAEImageProcessor" + ], + "model_classes": [ + "XCLIPModel" + ], + "sha": "299ffffc6b94c3558bf7dbc38e24074c99490046" }, "XGLMForCausalLM": { "tokenizer_classes": [ "XGLMTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFXGLMForCausalLM", + "XGLMForCausalLM" + ], + "sha": "95dc5c571119b6bf7d7c58e98897be4c724ca4ee" + }, + "XGLMModel": { + "tokenizer_classes": [ + "XGLMTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "TFXGLMModel", + "XGLMModel" + ], + "sha": "fcc8ad648ae1561cb4b35a1fd31c68b58296ef00" }, "XLMForMultipleChoice": { "tokenizer_classes": [ "XLMTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFXLMForMultipleChoice", + "XLMForMultipleChoice" + ], + "sha": "f0c8cc6462449ac9eb9b4158e433bd3c923db3af" }, "XLMForQuestionAnsweringSimple": { "tokenizer_classes": [ "XLMTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFXLMForQuestionAnsweringSimple", + "XLMForQuestionAnsweringSimple" + ], + "sha": "82e93a2653cf3646eaaf02d8cc5f8ff9a4551523" }, "XLMForSequenceClassification": { "tokenizer_classes": [ "XLMTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFXLMForSequenceClassification", + "XLMForSequenceClassification" + ], + "sha": "2d6892f5f703be9b481bca91477032bd0e36dbe5" }, "XLMForTokenClassification": { "tokenizer_classes": [ "XLMTokenizer" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFXLMForTokenClassification", + "XLMForTokenClassification" + ], + "sha": "9a591395e7a0643a03f5d2debb98caa3966e021c" + }, + "XLMModel": { + "tokenizer_classes": [ + "XLMTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "TFXLMModel", + "XLMModel" + ], + "sha": "022b86df246414ff712475d9ca55db690ff1d3bf" }, "XLMRobertaXLForCausalLM": { "tokenizer_classes": [ "XLMRobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "XLMRobertaXLForCausalLM" + ], + "sha": "54d7d901a31fcea581772a62a574f88ea994d931" }, "XLMRobertaXLForMaskedLM": { "tokenizer_classes": [ "XLMRobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "XLMRobertaXLForMaskedLM" + ], + "sha": "b9b8090515146aa703fc30da03f3c0995ad2ac58" }, "XLMRobertaXLForMultipleChoice": { "tokenizer_classes": [ "XLMRobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "XLMRobertaXLForMultipleChoice" + ], + "sha": "c6e7ef0a2d640d14f5b0333ffcd32fb2fb83312d" }, "XLMRobertaXLForQuestionAnswering": { "tokenizer_classes": [ "XLMRobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "XLMRobertaXLForQuestionAnswering" + ], + "sha": "4e433426b85bf8c1cea9261e91bd52d78d7bf397" }, "XLMRobertaXLForSequenceClassification": { "tokenizer_classes": [ "XLMRobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "XLMRobertaXLForSequenceClassification" + ], + "sha": "0b8361e84d0d75c0c7540e8dfcc339d37f683800" }, "XLMRobertaXLForTokenClassification": { "tokenizer_classes": [ "XLMRobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "XLMRobertaXLForTokenClassification" + ], + "sha": "70c4a472222415f8611c25ac7eea48944a6fa0aa" + }, + "XLMRobertaXLModel": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "XLMRobertaXLModel" + ], + "sha": "df4e5bb09f2fa1e5582dc9424882e544c8b28aed" + }, + "XLMWithLMHeadModel": { + "tokenizer_classes": [ + "XLMTokenizer" + ], + "processor_classes": [], + "model_classes": [ + "TFXLMWithLMHeadModel", + "XLMWithLMHeadModel" + ], + "sha": "db70bdefbaf095e88b8097e4b601d9105a511afa" }, "XLNetForMultipleChoice": { "tokenizer_classes": [ - "XLNetTokenizerFast", - "XLNetTokenizer" + "XLNetTokenizer", + "XLNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFXLNetForMultipleChoice", + "XLNetForMultipleChoice" + ], + "sha": "874306eaf380012b985180563bf140d604272f40" }, "XLNetForQuestionAnsweringSimple": { "tokenizer_classes": [ - "XLNetTokenizerFast", - "XLNetTokenizer" + "XLNetTokenizer", + "XLNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "TFXLNetForQuestionAnsweringSimple", + "XLNetForQuestionAnsweringSimple" ], - "processor_classes": [] + "sha": "8c6ac70e1946da872e5f93853247728d672a1dee" }, "XLNetForSequenceClassification": { "tokenizer_classes": [ - "XLNetTokenizerFast", - "XLNetTokenizer" + "XLNetTokenizer", + "XLNetTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "TFXLNetForSequenceClassification", + "XLNetForSequenceClassification" + ], + "sha": "3bba8894ba1d8e34cd807fd5a0f858449abf65b4" }, "XLNetForTokenClassification": { "tokenizer_classes": [ - "XLNetTokenizerFast", - "XLNetTokenizer" + "XLNetTokenizer", + "XLNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "TFXLNetForTokenClassification", + "XLNetForTokenClassification" + ], + "sha": "70598399e95e6df689fe5459ef9f016e55cc05ee" + }, + "XLNetLMHeadModel": { + "tokenizer_classes": [ + "XLNetTokenizer", + "XLNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "TFXLNetLMHeadModel", + "XLNetLMHeadModel" + ], + "sha": "fef32495d187c73201ba4e2854559bcc68e41e22" + }, + "XLNetModel": { + "tokenizer_classes": [ + "XLNetTokenizer", + "XLNetTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "TFXLNetModel", + "XLNetModel" + ], + "sha": "bebc65e9a3da5c0007713a61f6719293d361baa3" + }, + "XmodForCausalLM": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "XmodForCausalLM" + ], + "sha": "daf792dff8fef2f3d99eb5ee63b206032b3f69d7" + }, + "XmodForMaskedLM": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "XmodForMaskedLM" + ], + "sha": "b7c020898ef638ed4c2f420431b4efc0075c0e32" + }, + "XmodForMultipleChoice": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "XmodForMultipleChoice" + ], + "sha": "1fce9673e9f557d3204d504ebb86e285b20937f8" + }, + "XmodForQuestionAnswering": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "XmodForQuestionAnswering" + ], + "sha": "fc9ebdfeb481281375c25d585569404cc48b02da" + }, + "XmodForSequenceClassification": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "XmodForSequenceClassification" + ], + "sha": "bba89c4e18b6a29a1865462a75cb8bde12e7cc0c" + }, + "XmodForTokenClassification": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "XmodForTokenClassification" + ], + "sha": "efe3dad234d5a60f1165887d8b4bb1627e271508" + }, + "XmodModel": { + "tokenizer_classes": [ + "XLMRobertaTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "XmodModel" + ], + "sha": "62f3ca59e02a9637b12ce81566f1996c16fad872" }, "YolosForObjectDetection": { "tokenizer_classes": [], "processor_classes": [ - "YolosFeatureExtractor" - ] + "YolosImageProcessor" + ], + "model_classes": [ + "YolosForObjectDetection" + ], + "sha": "0a4aae25bfbe8b5edd4815cb00d697a6ba7d2126" + }, + "YolosModel": { + "tokenizer_classes": [], + "processor_classes": [ + "YolosImageProcessor" + ], + "model_classes": [ + "YolosModel" + ], + "sha": "339bc51f1914f031a550e5f95095ed4a4c22a7de" }, "YosoForMaskedLM": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "YosoForMaskedLM" + ], + "sha": "4ff7ab217e6d05d40ae8afd37df1a0bf5c6ab25f" }, "YosoForMultipleChoice": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "YosoForMultipleChoice" + ], + "sha": "c42bd94e4563cdfeb28e07aef80e801188706d9d" }, "YosoForQuestionAnswering": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "YosoForQuestionAnswering" + ], + "sha": "284f170b902c65ee257a2e8f255fa672cb4700c5" }, "YosoForSequenceClassification": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "YosoForSequenceClassification" + ], + "sha": "2e0a37a4dd12cee00eb15e4ecd0de4231a638966" }, "YosoForTokenClassification": { "tokenizer_classes": [ "AlbertTokenizerFast" ], - "processor_classes": [] + "processor_classes": [], + "model_classes": [ + "YosoForTokenClassification" + ], + "sha": "50dd3fe46f28f0efaacf8aa311b09f67b10edb74" + }, + "YosoModel": { + "tokenizer_classes": [ + "AlbertTokenizerFast" + ], + "processor_classes": [], + "model_classes": [ + "YosoModel" + ], + "sha": "6b42cca4d4f9204a3aa1197664663656719095cb" } -} +} \ No newline at end of file diff --git a/utils/create_dummy_models.py b/utils/create_dummy_models.py index 975c0703d4c4..9fb53df0ca26 100644 --- a/utils/create_dummy_models.py +++ b/utils/create_dummy_models.py @@ -27,7 +27,7 @@ from check_config_docstrings import get_checkpoint_from_config_class from datasets import load_dataset from get_test_info import get_model_to_tester_mapping, get_tester_classes_for_model -from huggingface_hub import Repository, create_repo, upload_folder +from huggingface_hub import Repository, create_repo, hf_api, upload_folder from transformers import ( CONFIG_MAPPING, @@ -70,6 +70,14 @@ INVALID_ARCH = [] TARGET_VOCAB_SIZE = 1024 +data = {"training_ds": None, "testing_ds": None} + +COMPOSITE_MODELS = { + "EncoderDecoderModel": "EncoderDecoderModel-bert-bert", + "SpeechEncoderDecoderModel": "SpeechEncoderDecoderModel-wav2vec2-bert", + "VisionEncoderDecoderModel": "VisionEncoderDecoderModel-vit-gpt2", + "VisionTextDualEncoderModel": "VisionTextDualEncoderModel-vit-bert", +} # This list contains the model architectures for which a tiny version could not be created. # Avoid to add new architectures here - unless we have verified carefully that it's (almost) impossible to create them. @@ -179,7 +187,7 @@ def _to_tuple(x): return processor_types -def get_architectures_from_config_class(config_class, arch_mappings): +def get_architectures_from_config_class(config_class, arch_mappings, models_to_skip=None): """Return a tuple of all possible architectures attributed to a configuration class `config_class`. For example, BertConfig -> [BertModel, BertForMaskedLM, ..., BertForQuestionAnswering]. @@ -192,12 +200,16 @@ def get_architectures_from_config_class(config_class, arch_mappings): # We avoid the duplication. architectures = set() + if models_to_skip is None: + models_to_skip = [] + models_to_skip = UNCONVERTIBLE_MODEL_ARCHITECTURES.union(models_to_skip) + for mapping in arch_mappings: if config_class in mapping: models = mapping[config_class] models = tuple(models) if isinstance(models, collections.abc.Sequence) else (models,) for model in models: - if model.__name__ not in UNCONVERTIBLE_MODEL_ARCHITECTURES: + if model.__name__ not in models_to_skip: architectures.add(model) architectures = tuple(architectures) @@ -422,11 +434,13 @@ def get_tiny_config(config_class, model_class=None, **model_tester_kwargs): def convert_tokenizer(tokenizer_fast: PreTrainedTokenizerFast): - new_tokenizer = tokenizer_fast.train_new_from_iterator(training_ds["text"], TARGET_VOCAB_SIZE, show_progress=False) + new_tokenizer = tokenizer_fast.train_new_from_iterator( + data["training_ds"]["text"], TARGET_VOCAB_SIZE, show_progress=False + ) # Make sure it at least runs if not isinstance(new_tokenizer, LayoutLMv3TokenizerFast): - new_tokenizer(testing_ds["text"]) + new_tokenizer(data["testing_ds"]["text"]) return new_tokenizer @@ -640,16 +654,17 @@ def fill_result_with_error(result, error, trace, models_to_create): result["processor"] = {p.__class__.__name__: p.__class__.__name__ for p in result["processor"].values()} -def upload_model(model_dir, organization): +def upload_model(model_dir, organization, token): """Upload the tiny models""" arch_name = model_dir.split(os.path.sep)[-1] repo_name = f"tiny-random-{arch_name}" + repo_id = f"{organization}/{repo_name}" repo_exist = False error = None try: - create_repo(repo_id=f"{organization}/{repo_name}", exist_ok=False, repo_type="model") + create_repo(repo_id=repo_id, exist_ok=False, repo_type="model", token=token) except Exception as e: error = e if "You already created" in str(e): @@ -657,14 +672,14 @@ def upload_model(model_dir, organization): logger.warning("Remote repository exists and will be cloned.") repo_exist = True try: - create_repo(repo_id=repo_name, organization=organization, exist_ok=True, repo_type="model") + create_repo(repo_id=repo_id, exist_ok=True, repo_type="model", token=token) except Exception as e: error = e if error is not None: raise error with tempfile.TemporaryDirectory() as tmpdir: - repo = Repository(local_dir=tmpdir, clone_from=f"{organization}/{repo_name}") + repo = Repository(local_dir=tmpdir, clone_from=repo_id, token=token) repo.git_pull() shutil.copytree(model_dir, tmpdir, dirs_exist_ok=True) @@ -672,19 +687,21 @@ def upload_model(model_dir, organization): # Open a PR on the existing Hub repo. hub_pr_url = upload_folder( folder_path=model_dir, - repo_id=f"{organization}/{repo_name}", + repo_id=repo_id, repo_type="model", commit_message=f"Update tiny models for {arch_name}", commit_description=f"Upload tiny models for {arch_name}", create_pr=True, + token=token, ) logger.warning(f"PR open in {hub_pr_url}.") + # TODO: We need this information? else: # Push to Hub repo directly repo.git_add(auto_lfs_track=True) repo.git_commit(f"Upload tiny models for {arch_name}") repo.git_push(blocking=True) # this prints a progress bar with the upload - logger.warning(f"Tiny models {arch_name} pushed to {organization}/{repo_name}.") + logger.warning(f"Tiny models {arch_name} pushed to {repo_id}.") def build_composite_models(config_class, output_dir): @@ -704,6 +721,7 @@ def build_composite_models(config_class, output_dir): SpeechEncoderDecoderModel, TFEncoderDecoderModel, TFVisionEncoderDecoderModel, + TFVisionTextDualEncoderModel, VisionEncoderDecoderModel, VisionTextDualEncoderModel, ViTConfig, @@ -753,7 +771,7 @@ def build_composite_models(config_class, output_dir): encoder_class = ViTModel decoder_class = BertModel model_class = VisionTextDualEncoderModel - tf_model_class = None + tf_model_class = TFVisionTextDualEncoderModel with tempfile.TemporaryDirectory() as tmpdir: try: @@ -1097,13 +1115,14 @@ def build(config_class, models_to_create, output_dir): return result -def build_tiny_model_summary(results): +def build_tiny_model_summary(results, organization=None, token=None): """Build a summary: a dictionary of the form { model architecture name: { "tokenizer_classes": [...], - "processor_classes": [...] + "processor_classes": [...], + "model_classes": [...], } .. } @@ -1111,19 +1130,42 @@ def build_tiny_model_summary(results): tiny_model_summary = {} for config_name in results: processors = [key for key, value in results[config_name]["processor"].items()] - tokenizer_classes = [x for x in processors if x.endswith("TokenizerFast") or x.endswith("Tokenizer")] - processor_classes = [x for x in processors if x not in tokenizer_classes] + tokenizer_classes = sorted([x for x in processors if x.endswith("TokenizerFast") or x.endswith("Tokenizer")]) + processor_classes = sorted([x for x in processors if x not in tokenizer_classes]) for framework in FRAMEWORKS: if framework not in results[config_name]: continue for arch_name in results[config_name][framework]: + model_classes = [arch_name] + base_arch_name = arch_name[2:] if arch_name.startswith("TF") else arch_name # tiny model is not created for `arch_name` - if results[config_name][framework][arch_name] is None: - continue - tiny_model_summary[arch_name] = { - "tokenizer_classes": tokenizer_classes, - "processor_classes": processor_classes, - } + if results[config_name][framework][arch_name]["model"] is None: + model_classes = [] + if base_arch_name not in tiny_model_summary: + tiny_model_summary[base_arch_name] = {} + tiny_model_summary[base_arch_name].update( + { + "tokenizer_classes": tokenizer_classes, + "processor_classes": processor_classes, + } + ) + tiny_model_summary[base_arch_name]["model_classes"] = sorted( + tiny_model_summary[base_arch_name].get("model_classes", []) + model_classes + ) + if organization is not None: + repo_name = f"tiny-random-{base_arch_name}" + # composite models' checkpoints have more precise repo. names on the Hub. + if base_arch_name in COMPOSITE_MODELS: + repo_name = f"tiny-random-{COMPOSITE_MODELS[base_arch_name]}" + repo_id = f"{organization}/{repo_name}" + try: + commit_hash = hf_api.repo_info(repo_id, token=token).sha + except Exception: + # The directory is not created, but processor(s) is/are included in `results`. + logger.warning(f"Failed to get information for {repo_id}.\n{traceback.format_exc()}") + del tiny_model_summary[base_arch_name] + continue + tiny_model_summary[base_arch_name]["sha"] = commit_hash return tiny_model_summary @@ -1176,11 +1218,23 @@ def build_simple_report(results): return text, failed_text -if __name__ == "__main__": +def create_tiny_models( + output_path, + all, + model_types, + models_to_skip, + no_check, + upload, + organization, + token, +): clone_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) if os.getcwd() != clone_path: raise ValueError(f"This script should be run from the root of the clone of `transformers` {clone_path}") + report_path = os.path.join(output_path, "reports") + os.makedirs(report_path) + _pytorch_arch_mappings = [ x for x in dir(transformers_module) @@ -1189,118 +1243,145 @@ def build_simple_report(results): _tensorflow_arch_mappings = [ x for x in dir(transformers_module) if x.startswith("TF_MODEL_") and x.endswith("_MAPPING") ] - # _flax_arch_mappings = [x for x in dir(transformers_module) if x.startswith("FLAX_MODEL_") and x.endswith("_MAPPING")] pytorch_arch_mappings = [getattr(transformers_module, x) for x in _pytorch_arch_mappings] tensorflow_arch_mappings = [getattr(transformers_module, x) for x in _tensorflow_arch_mappings] - # flax_arch_mappings = [getattr(transformers_module, x) for x in _flax_arch_mappings] ds = load_dataset("wikitext", "wikitext-2-raw-v1") - training_ds = ds["train"] - testing_ds = ds["test"] - - def list_str(values): - return values.split(",") - - parser = argparse.ArgumentParser() - parser.add_argument("--all", action="store_true", help="Will create all tiny models.") - parser.add_argument( - "--no_check", - action="store_true", - help="If set, will not check the validity of architectures. Use with caution.", - ) - parser.add_argument( - "-m", - "--model_types", - type=list_str, - help="Comma-separated list of model type(s) from which the tiny models will be created.", - ) - parser.add_argument("--upload", action="store_true", help="If to upload the created tiny models to the Hub.") - parser.add_argument( - "--organization", - default=None, - type=str, - help="The organization on the Hub to which the tiny models will be uploaded.", - ) - parser.add_argument("output_path", type=Path, help="Path indicating where to store generated model.") - - args = parser.parse_args() - - if not args.all and not args.model_types: - raise ValueError("Please provide at least one model type or pass `--all` to export all architectures.") + data["training_ds"] = ds["train"] + data["testing_ds"] = ds["test"] config_classes = CONFIG_MAPPING.values() - if not args.all: - config_classes = [CONFIG_MAPPING[model_type] for model_type in args.model_types] + if not all: + config_classes = [CONFIG_MAPPING[model_type] for model_type in model_types] # A map from config classes to tuples of processors (tokenizer, feature extractor, processor) classes processor_type_map = {c: get_processor_types_from_config_class(c) for c in config_classes} - to_create = { - c: { - "processor": processor_type_map[c], - "pytorch": get_architectures_from_config_class(c, pytorch_arch_mappings), - "tensorflow": get_architectures_from_config_class(c, tensorflow_arch_mappings), - # "flax": get_architectures_from_config_class(c, flax_arch_mappings), - } - for c in config_classes - } + to_create = {} + for c in config_classes: + processors = processor_type_map[c] + models = get_architectures_from_config_class(c, pytorch_arch_mappings, models_to_skip) + tf_models = get_architectures_from_config_class(c, tensorflow_arch_mappings, models_to_skip) + if len(models) + len(tf_models) > 0: + to_create[c] = {"processor": processors, "pytorch": models, "tensorflow": tf_models} results = {} for c, models_to_create in list(to_create.items()): print(f"Create models for {c.__name__} ...") - result = build(c, models_to_create, output_dir=os.path.join(args.output_path, c.model_type)) + result = build(c, models_to_create, output_dir=os.path.join(output_path, c.model_type)) results[c.__name__] = result print("=" * 40) - with open("tiny_model_creation_report.json", "w") as fp: - json.dump(results, fp, indent=4) + if upload: + if organization is None: + raise ValueError("The argument `organization` could not be `None`. No model is uploaded") + + to_upload = [] + for model_type in os.listdir(output_path): + # This is the directory containing the reports + if model_type == "reports": + continue + for arch in os.listdir(os.path.join(output_path, model_type)): + if arch == "processors": + continue + to_upload.append(os.path.join(output_path, model_type, arch)) + to_upload = sorted(to_upload) + + upload_results = {} + if len(to_upload) > 0: + for model_dir in to_upload: + try: + upload_model(model_dir, organization, token) + except Exception as e: + error = f"Failed to upload {model_dir}. {e.__class__.__name__}: {e}" + logger.error(error) + upload_results[model_dir] = error + + with open(os.path.join(report_path, "failed_uploads.json"), "w") as fp: + json.dump(upload_results, fp, indent=4) # Build the tiny model summary file. The `tokenizer_classes` and `processor_classes` could be both empty lists. # When using the items in this file to update the file `tests/utils/tiny_model_summary.json`, the model # architectures with `tokenizer_classes` and `processor_classes` being both empty should **NOT** be added to # `tests/utils/tiny_model_summary.json`. - tiny_model_summary = build_tiny_model_summary(results) - with open("tiny_model_summary.json", "w") as fp: + tiny_model_summary = build_tiny_model_summary(results, organization=organization, token=token) + with open(os.path.join(report_path, "tiny_model_summary.json"), "w") as fp: json.dump(tiny_model_summary, fp, indent=4) + with open(os.path.join(report_path, "tiny_model_creation_report.json"), "w") as fp: + json.dump(results, fp, indent=4) + # Build the warning/failure report (json format): same format as the complete `results` except this contains only # warnings or errors. failed_results = build_failed_report(results) - with open("failed_report.json", "w") as fp: + with open(os.path.join(report_path, "failed_report.json"), "w") as fp: json.dump(failed_results, fp, indent=4) simple_report, failed_report = build_simple_report(results) # The simplified report: a .txt file with each line of format: # {model architecture name}: {OK or error message} - with open("simple_report.txt", "w") as fp: + with open(os.path.join(report_path, "simple_report.txt"), "w") as fp: fp.write(simple_report) # The simplified failure report: same above except this only contains line with errors - with open("simple_failed_report.txt", "w") as fp: + with open(os.path.join(report_path, "simple_failed_report.txt"), "w") as fp: fp.write(failed_report) - if args.upload: - if args.organization is None: - raise ValueError("The argument `organization` could not be `None`. No model is uploaded") - to_upload = [] - for model_type in os.listdir(args.output_path): - for arch in os.listdir(os.path.join(args.output_path, model_type)): - if arch == "processors": - continue - to_upload.append(os.path.join(args.output_path, model_type, arch)) - to_upload = sorted(to_upload) +if __name__ == "__main__": + ds = load_dataset("wikitext", "wikitext-2-raw-v1") + training_ds = ds["train"] + testing_ds = ds["test"] - upload_results = {} - if len(to_upload) > 0: - for model_dir in to_upload: - try: - upload_model(model_dir, args.organization) - except Exception as e: - error = f"Failed to upload {model_dir}. {e.__class__.__name__}: {e}" - logger.error(error) - upload_results[model_dir] = error + def list_str(values): + return values.split(",") - with open("failed_uploads.json", "w") as fp: - json.dump(upload_results, fp, indent=4) + parser = argparse.ArgumentParser() + parser.add_argument("--all", action="store_true", help="Will create all tiny models.") + parser.add_argument( + "--no_check", + action="store_true", + help="If set, will not check the validity of architectures. Use with caution.", + ) + parser.add_argument( + "-m", + "--model_types", + type=list_str, + help="Comma-separated list of model type(s) from which the tiny models will be created.", + ) + parser.add_argument( + "--models_to_skip", + type=list_str, + help=( + "Comma-separated list of model class names(s) from which the tiny models won't be created.\nThis is usually" + "the list of model classes that have their tiny versions already uploaded to the Hub." + ), + ) + parser.add_argument("--upload", action="store_true", help="If to upload the created tiny models to the Hub.") + parser.add_argument( + "--organization", + default=None, + type=str, + help="The organization on the Hub to which the tiny models will be uploaded.", + ) + parser.add_argument( + "--token", default=None, type=str, help="A valid authentication token for HuggingFace Hub with write access." + ) + parser.add_argument("output_path", type=Path, help="Path indicating where to store generated model.") + + args = parser.parse_args() + + if not args.all and not args.model_types: + raise ValueError("Please provide at least one model type or pass `--all` to export all architectures.") + + create_tiny_models( + args.output_path, + args.all, + args.model_types, + args.models_to_skip, + args.no_check, + args.upload, + args.organization, + args.token, + ) diff --git a/utils/update_tiny_models.py b/utils/update_tiny_models.py new file mode 100644 index 000000000000..1b53737ed732 --- /dev/null +++ b/utils/update_tiny_models.py @@ -0,0 +1,219 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A script running `create_dummy_models.py` with a pre-defined set of arguments. + +This file is intended to be used in a CI workflow file without the need of specifying arguments. It creates and uploads +tiny models for all model classes (if their tiny versions are not on the Hub yet), as well as produces an updated +version of `tests/utils/tiny_model_summary.json`. That updated file should be merged into the `main` branch of +`transformers` so the pipeline testing will use the latest created/updated tiny models. +""" + + +import copy +import json +import os +import time + +from create_dummy_models import COMPOSITE_MODELS, create_tiny_models +from huggingface_hub import ModelFilter, hf_api + +import transformers +from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer +from transformers.image_processing_utils import BaseImageProcessor + + +def get_all_model_names(): + model_names = set() + # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way. + for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]: + module = getattr(transformers.models.auto, module_name, None) + if module is None: + continue + # all mappings in a single auto modeling file + mapping_names = [ + x + for x in dir(module) + if x.endswith("_MAPPING_NAMES") + and (x.startswith("MODEL_") or x.startswith("TF_MODEL_") or x.startswith("FLAX_MODEL_")) + ] + for name in mapping_names: + mapping = getattr(module, name) + if mapping is not None: + for v in mapping.values(): + if isinstance(v, (list, tuple)): + model_names.update(v) + elif isinstance(v, str): + model_names.add(v) + + return sorted(model_names) + + +def get_tiny_model_names_from_repo(): + # All model names defined in auto mappings + model_names = set(get_all_model_names()) + + with open("tests/utils/tiny_model_summary.json") as fp: + tiny_model_info = json.load(fp) + tiny_models_names = set() + for model_base_name in tiny_model_info: + tiny_models_names.update(tiny_model_info[model_base_name]["model_classes"]) + + # Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub. + not_on_hub = model_names.difference(tiny_models_names) + for model_name in copy.copy(tiny_models_names): + if not model_name.startswith("TF") and f"TF{model_name}" in not_on_hub: + tiny_models_names.remove(model_name) + elif model_name.startswith("TF") and model_name[2:] in not_on_hub: + tiny_models_names.remove(model_name) + + return sorted(tiny_models_names) + + +def get_tiny_model_summary_from_hub(output_path): + special_models = COMPOSITE_MODELS.values() + + # All tiny model base names on Hub + model_names = get_all_model_names() + models = hf_api.list_models( + filter=ModelFilter( + author="hf-internal-testing", + ) + ) + _models = set() + for x in models: + model = x.modelId + org, model = model.split("/") + if not model.startswith("tiny-random-"): + continue + model = model.replace("tiny-random-", "") + if not model[0].isupper(): + continue + if model not in model_names and model not in special_models: + continue + _models.add(model) + + models = sorted(_models) + # All tiny model names on Hub + summary = {} + for model in models: + repo_id = f"hf-internal-testing/tiny-random-{model}" + model = model.split("-")[0] + try: + repo_info = hf_api.repo_info(repo_id) + content = { + "tokenizer_classes": set(), + "processor_classes": set(), + "model_classes": set(), + "sha": repo_info.sha, + } + except Exception: + continue + try: + time.sleep(1) + tokenizer_fast = AutoTokenizer.from_pretrained(repo_id) + content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__) + except Exception: + pass + try: + time.sleep(1) + tokenizer_slow = AutoTokenizer.from_pretrained(repo_id, use_fast=False) + content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__) + except Exception: + pass + try: + time.sleep(1) + img_p = AutoImageProcessor.from_pretrained(repo_id) + content["processor_classes"].add(img_p.__class__.__name__) + except Exception: + pass + try: + time.sleep(1) + feat_p = AutoFeatureExtractor.from_pretrained(repo_id) + if not isinstance(feat_p, BaseImageProcessor): + content["processor_classes"].add(feat_p.__class__.__name__) + except Exception: + pass + try: + time.sleep(1) + model_class = getattr(transformers, model) + m = model_class.from_pretrained(repo_id) + content["model_classes"].add(m.__class__.__name__) + except Exception: + pass + try: + time.sleep(1) + model_class = getattr(transformers, f"TF{model}") + m = model_class.from_pretrained(repo_id) + content["model_classes"].add(m.__class__.__name__) + except Exception: + pass + + content["tokenizer_classes"] = sorted(content["tokenizer_classes"]) + content["processor_classes"] = sorted(content["processor_classes"]) + content["model_classes"] = sorted(content["model_classes"]) + + summary[model] = content + with open(os.path.join(output_path, "hub_tiny_model_summary.json"), "w") as fp: + json.dump(summary, fp, ensure_ascii=False, indent=4) + + +def update_tiny_model_summary_file(report_path): + with open(os.path.join(report_path, "tiny_model_summary.json")) as fp: + new_data = json.load(fp) + with open("tests/utils/tiny_model_summary.json") as fp: + data = json.load(fp) + for key, value in new_data.items(): + if key not in data: + data[key] = value + else: + for attr in ["tokenizer_classes", "processor_classes", "model_classes"]: + # we might get duplication here. We will remove them below when creating `updated_data`. + data[key][attr].extend(value[attr]) + new_sha = value["sha"] + if new_sha is not None: + data[key]["sha"] = new_sha + + updated_data = {} + for key in sorted(data.keys()): + updated_data[key] = {} + for attr, value in data[key].items(): + # deduplication and sort + updated_data[key][attr] = sorted(set(value)) if attr != "sha" else value + + with open(os.path.join(report_path, "updated_tiny_model_summary.json"), "w") as fp: + json.dump(updated_data, fp, indent=4, ensure_ascii=False) + + +if __name__ == "__main__": + output_path = "tiny_models" + all = True + model_types = None + models_to_skip = get_tiny_model_names_from_repo() + no_check = True + upload = True + organization = "hf-internal-testing" + + create_tiny_models( + output_path, + all, + model_types, + models_to_skip, + no_check, + upload, + organization, + token=os.environ.get("TOKEN", None), + ) + + update_tiny_model_summary_file(report_path=os.path.join(output_path, "reports")) From 7e64776a0d712df73d52491f5b62a99a05877f50 Mon Sep 17 00:00:00 2001 From: James Reed <6jamesr6@gmail.com> Date: Fri, 24 Mar 2023 04:04:51 -0700 Subject: [PATCH 181/277] [HFTracer] Make embeddings ops take on the dtype of the weight (#22347) * [HFTracer] Make embeddings ops take on the dtype of the weight * fix bug --- src/transformers/utils/fx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/utils/fx.py b/src/transformers/utils/fx.py index 350d9e91245a..a9591f8ffeb3 100755 --- a/src/transformers/utils/fx.py +++ b/src/transformers/utils/fx.py @@ -171,13 +171,13 @@ def _generate_supported_model_class_names( def torch_nn_embedding(self, input): - return torch.empty(*input.shape, self.weight.shape[-1], device="meta") + return torch.empty(*input.shape, self.weight.shape[-1], device="meta", dtype=self.weight.dtype) def torch_nn_functional_embedding( input, weight, padding_idx=None, max_norm=None, norm_type=2.0, scale_grad_by_freq=False, sparse=False ): - return torch.empty(*input.shape, weight.shape[-1], device="meta") + return torch.empty(*input.shape, weight.shape[-1], device="meta", dtype=weight.dtype) def torch_nn_layernorm(self, input): From 4e53190e3d54002354e7545a6f1f51b75bbe3e6e Mon Sep 17 00:00:00 2001 From: Ashwin Mathur <97467100+awinml@users.noreply.github.com> Date: Fri, 24 Mar 2023 17:02:18 +0530 Subject: [PATCH 182/277] Fix typo in Greedy Search Description (#22345) Fix typo in greedy search docs --- docs/source/en/generation_strategies.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/en/generation_strategies.mdx b/docs/source/en/generation_strategies.mdx index 2e3671fe63ba..44692169405f 100644 --- a/docs/source/en/generation_strategies.mdx +++ b/docs/source/en/generation_strategies.mdx @@ -149,7 +149,6 @@ Here, we'll show some of the parameters that control the decoding strategies and ### Greedy Search [`generate`] uses greedy search decoding by default so you don't have to pass any parameters to enable it. This means the parameters `num_beams` is set to 1 and `do_sample=False`. -`do_sample=False`. Because it is a default strategy, you do not have to pass any parameters to `generate()` method to enable it. ```python >>> from transformers import AutoModelForCausalLM, AutoTokenizer @@ -303,4 +302,4 @@ The groups are selected to ensure they are distinct enough compared to the other This guide illustrates the main parameters that enable various decoding strategies. More advanced parameters exist for the [`generate`] method, which gives you even further control over the [`generate`] method's behavior. -For the complete list of the available parameters, refer to the [API documentation](./main_classes/text_generation.mdx). \ No newline at end of file +For the complete list of the available parameters, refer to the [API documentation](./main_classes/text_generation.mdx). From 75a00cffd9e1d89186c3f8669b9a166bf03d97f2 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Fri, 24 Mar 2023 11:33:16 +0000 Subject: [PATCH 183/277] Generate: Add GPTNeoX integration test (#22346) --- .../models/gpt_neox/test_modeling_gpt_neox.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/models/gpt_neox/test_modeling_gpt_neox.py b/tests/models/gpt_neox/test_modeling_gpt_neox.py index 89765e05610b..1798f01358ff 100644 --- a/tests/models/gpt_neox/test_modeling_gpt_neox.py +++ b/tests/models/gpt_neox/test_modeling_gpt_neox.py @@ -17,8 +17,8 @@ import unittest -from transformers import GPTNeoXConfig, is_torch_available -from transformers.testing_utils import require_torch, torch_device +from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available +from transformers.testing_utils import require_torch, slow, torch_device from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester @@ -232,3 +232,28 @@ def test_model_for_causal_lm(self): @unittest.skip(reason="Feed forward chunking is not implemented") def test_feed_forward_chunking(self): pass + + +@require_torch +class GPTNeoXLanguageGenerationTest(unittest.TestCase): + @slow + def test_lm_generate_codegen(self): + tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped") + for checkpointing in [True, False]: + model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped") + + if checkpointing: + model.gradient_checkpointing_enable() + else: + model.gradient_checkpointing_disable() + model.to(torch_device) + + inputs = tokenizer("My favorite food is", return_tensors="pt").to(torch_device) + expected_output = ( + "My favorite food is the chicken and rice.\n\nI love to cook and bake. I love to cook and bake" + ) + + output_ids = model.generate(**inputs, do_sample=False, max_new_tokens=20) + output_str = tokenizer.batch_decode(output_ids)[0] + + self.assertEqual(output_str, expected_output) From f876ad9a42f1354298e1638c5a0611f306184103 Mon Sep 17 00:00:00 2001 From: Mitch Naylor Date: Fri, 24 Mar 2023 07:17:27 -0500 Subject: [PATCH 184/277] Add Mega: Moving Average Equipped Gated Attention (#21766) * add mega file structure and plain pytorch version of mega source code * added config class with old naming conventions * filled in mega documentation * added config class and embeddings with optional token types * updated notes * starting the conversion process, deleted intermediate and added use_cache back to config * renamed config attributes in modeling_mega.py * checkpointing before refactoring incremental decoding functions * removed stateful incremental key/values for EMA and self-attention * refactored MovingAverageGatedAttention to remove stateful k/v history and use unified attention mask * MovingAverageGatedAttention works with incremental decoding + past values, added sequence length enforcement * more comments in MovingAverageGatedAttention + checkpointing before GatedCrossAttention * bug fix in attention mask handling in MovingAverageGatedAttention * removed incremental state from GatedCrossAttention and removed IncrementalState class * finished gated cross attention and got MegaLayer working * fixed causal masking in mega decoder * fixed how padding and causal masks are passed through MegaLayer with and without k/v caching * finished MegaModel; tested with encoder, decoder-only, and cross-attention type inputs; started work on downstream classes; removed mentions of position_ids * added optional dense hidden layer for masked and causal LM classes * docstring updates in MultiHeadEMA and GatedCrossAttention, removed unnecessary inputs in cross-attention * removed before_attn_fn in Mega class and updated docstrings and comments up to there * bug fix in MovingAverageGatedAttention masking * working conversion of MLM checkpoint in scratchpad script -- perfect matches * moved arg for hidden dense layer in LM head to config; discovered issue where from_pretrained is renaming gamma and beta parameters * renamed gamma and beta parameters to avoid HF renaming when loading from checkpoint * finished checkpoint conversion script * cleanup old class in mega config script * removed 'copied from' statements and passing integration tests * added num_attention_heads=1 to config for integration compatibility, decoder tests working, generation tests failing * fixed tuple output of megamodel * all common tests passing after fixing issues in decoder, gradient retention, and initialization * added mega-specific tests, ready for more documentation and style checks * updated docstrings; checkpoint before style fixes * style and quality checks, fixed initialization problem in float_tensor, ready for PR * added mega to toctree * removed unnecessary arg in megaconfig * removed unused arg and fixed code samples with leftover roberta models * Apply suggestions from code review Applied all suggestions except the one renaming a class, as I'll need to update that througout Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * fixed issue where .view breaks batch dimension, conversion script fixed with absolute imports, updated readme with Mega->MEGA * removed asserts in Mega code, renamed sequencenorm, gatedcrossattention, and NFFN, replaced get_activation_fn with ACTFN, and added sequencenorm to layer norms * reformatted .forward() docstrings to match style and removed unused mask input in cross-attention * removed all reset_parameters() methods and rolled into MegaPreTrainedModel._init_weights() * renamed all single-letter variables and improved readability in tensor size comments, Mega->MEGA in 2 documentation files * variable names in NFFN * manual Mega->MEGA changes in docs * Mega->MEGA in config auto * style and quality fixes * Apply suggestions from code review Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * renamed parameters and variables with confusing names, added copied from statements, moved fft conv to its own method, other cleanup from PR comments * commit before dealing with merge conflicts * made new attention activation functions available in ACT2FN and added generation test from OPT * style and quality in activations and tests * documentation fixes, renaming variables in dropout and rotary positions, used built-in causal masking, encoders->layers in MegaModel, moved comments into docstrings * style and quality fixes after latest updates, before rotary position ids * causal mask in MegaBlock docstring + added missing device passing * Apply suggestions from code review Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> * Update README.md Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * added Mega prefixes where missing, reverted MegaSequenceNorm to if-else, other module renaming requested in PR * style and quality fixes + readme updates pointing to main --------- Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- README.md | 1 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/en/_toctree.yml | 2 + docs/source/en/index.mdx | 2 + docs/source/en/model_doc/mega.mdx | 78 + docs/source/en/serialization.mdx | 1 + docs/source/en/tasks/language_modeling.mdx | 2 +- .../en/tasks/masked_language_modeling.mdx | 2 +- docs/source/en/tasks/multiple_choice.mdx | 2 +- docs/source/en/tasks/question_answering.mdx | 2 +- .../en/tasks/sequence_classification.mdx | 2 +- docs/source/en/tasks/token_classification.mdx | 2 +- src/transformers/__init__.py | 26 + src/transformers/activations.py | 43 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + src/transformers/models/auto/modeling_auto.py | 9 + .../models/auto/tokenization_auto.py | 1 + src/transformers/models/mega/__init__.py | 70 + .../models/mega/configuration_mega.py | 242 ++ ..._original_pytorch_checkpoint_to_pytorch.py | 291 +++ src/transformers/models/mega/modeling_mega.py | 2300 +++++++++++++++++ src/transformers/utils/dummy_pt_objects.py | 59 + tests/models/mega/__init__.py | 0 tests/models/mega/test_modeling_mega.py | 649 +++++ 30 files changed, 3790 insertions(+), 6 deletions(-) create mode 100644 docs/source/en/model_doc/mega.mdx create mode 100644 src/transformers/models/mega/__init__.py create mode 100644 src/transformers/models/mega/configuration_mega.py create mode 100644 src/transformers/models/mega/convert_mega_original_pytorch_checkpoint_to_pytorch.py create mode 100644 src/transformers/models/mega/modeling_mega.py create mode 100644 tests/models/mega/__init__.py create mode 100644 tests/models/mega/test_modeling_mega.py diff --git a/README.md b/README.md index 209d3e21b233..2995b1b9c592 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://arxiv.org/abs/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov. 1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. +1. **[MEGA](https://huggingface.co/docs/transformers/main/model_doc/mega)** (from Facebook) released with the paper [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) by Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. diff --git a/README_es.md b/README_es.md index b350d45d40b8..c37dd68829da 100644 --- a/README_es.md +++ b/README_es.md @@ -365,6 +365,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://arxiv.org/abs/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov. 1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. +1. **[MEGA](https://huggingface.co/docs/transformers/main/model_doc/mega)** (from Facebook) released with the paper [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) by Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. diff --git a/README_hd.md b/README_hd.md index 71bf9b61b59c..447e144b7b3b 100644 --- a/README_hd.md +++ b/README_hd.md @@ -337,6 +337,7 @@ conda install -c huggingface transformers 1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (मेटा और UIUC से) पेपर के साथ जारी किया गया [प्रति-पिक्सेल वर्गीकरण वह सब नहीं है जिसकी आपको सिमेंटिक सेगमेंटेशन की आवश्यकता है] (https://arxiv.org/abs/2107.06278) बोवेन चेंग, अलेक्जेंडर जी. श्विंग, अलेक्जेंडर किरिलोव द्वारा >>>>>> रिबेस ठीक करें 1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (फेसबुक से) साथ में पेपर [न्यूरल मशीन ट्रांसलेशन के लिए मल्टीलिंगुअल डीनोइजिंग प्री-ट्रेनिंग](https://arxiv. org/abs/2001.08210) यिनहान लियू, जियाताओ गु, नमन गोयल, जियान ली, सर्गेई एडुनोव, मार्जन ग़ज़विनिनेजाद, माइक लुईस, ल्यूक ज़ेटलमॉयर द्वारा। 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (फेसबुक से) साथ में पेपर [एक्स्टेंसिबल बहुभाषी प्रीट्रेनिंग और फाइनट्यूनिंग के साथ बहुभाषी अनुवाद](https://arxiv युकिंग टैंग, चाउ ट्रान, जियान ली, पेंग-जेन चेन, नमन गोयल, विश्रव चौधरी, जियाताओ गु, एंजेला फैन द्वारा .org/abs/2008.00401)। +1. **[MEGA](https://huggingface.co/docs/transformers/main/model_doc/mega)** (Facebook से) Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer. द्वाराअनुसंधान पत्र [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) के साथ जारी किया गया 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA से) कागज के साथ [Megatron-LM: मॉडल का उपयोग करके बहु-अरब पैरामीटर भाषा मॉडल का प्रशिक्षण Parallelism](https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा। 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA से) साथ वाला पेपर [Megatron-LM: ट्रेनिंग मल्टी-बिलियन पैरामीटर लैंग्वेज मॉडल्स यूजिंग मॉडल पैरेललिज़्म] (https://arxiv.org/abs/1909.08053) मोहम्मद शोएबी, मोस्टोफा पटवारी, राउल पुरी, पैट्रिक लेग्रेस्ले, जेरेड कैस्पर और ब्रायन कैटानज़ारो द्वारा पोस्ट किया गया। 1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research से) Peng Wang, Cheng Da, and Cong Yao. द्वाराअनुसंधान पत्र [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) के साथ जारी किया गया diff --git a/README_ja.md b/README_ja.md index 75f45c4462a4..5ae781aad4f1 100644 --- a/README_ja.md +++ b/README_ja.md @@ -399,6 +399,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (Meta and UIUC から) Bowen Cheng, Alexander G. Schwing, Alexander Kirillov から公開された研究論文: [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://arxiv.org/abs/2107.06278) 1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook から) Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer から公開された研究論文: [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook から) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan から公開された研究論文: [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) +1. **[MEGA](https://huggingface.co/docs/transformers/main/model_doc/mega)** (Facebook から) Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer. から公開された研究論文 [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA から) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro から公開された研究論文: [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research から) Peng Wang, Cheng Da, and Cong Yao. から公開された研究論文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) diff --git a/README_ko.md b/README_ko.md index 703c898707fa..c87edecdb13d 100644 --- a/README_ko.md +++ b/README_ko.md @@ -314,6 +314,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (Meta and UIUC 에서) Bowen Cheng, Alexander G. Schwing, Alexander Kirillov 의 [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://arxiv.org/abs/2107.06278) 논문과 함께 발표했습니다. 1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook 에서) Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer 의 [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) 논문과 함께 발표했습니다. 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (Facebook 에서) Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 의 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 논문과 함께 발표했습니다. +1. **[MEGA](https://huggingface.co/docs/transformers/main/model_doc/mega)** (Facebook 에서 제공)은 Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer.의 [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655)논문과 함께 발표했습니다. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (NVIDIA 에서) Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 의 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 논문과 함께 발표했습니다. 1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (Alibaba Research 에서 제공)은 Peng Wang, Cheng Da, and Cong Yao.의 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592)논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index 8fa869b498c7..b342dd4fae97 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -338,6 +338,7 @@ conda install -c huggingface transformers 1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://arxiv.org/abs/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov 1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (来自 Facebook) 伴随论文 [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) 由 Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer 发布。 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (来自 Facebook) 伴随论文 [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) 由 Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan 发布。 +1. **[MEGA](https://huggingface.co/docs/transformers/main/model_doc/mega)** (来自 Facebook) 伴随论文 [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) 由 Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer 发布。 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (来自 NVIDIA) 伴随论文 [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) 由 Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro 发布。 1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (来自 Alibaba Research) 伴随论文 [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) 由 Peng Wang, Cheng Da, and Cong Yao 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index eae33654a950..ea0f4e1047c5 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -350,6 +350,7 @@ conda install -c huggingface transformers 1. **[MaskFormer](https://huggingface.co/docs/transformers/model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://arxiv.org/abs/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov 1. **[mBART](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. 1. **[mBART-50](https://huggingface.co/docs/transformers/model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. +1. **[MEGA](https://huggingface.co/docs/transformers/main/model_doc/mega)** (from Facebook) released with the paper [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) by Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer. 1. **[Megatron-BERT](https://huggingface.co/docs/transformers/model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](https://huggingface.co/docs/transformers/model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[MGP-STR](https://huggingface.co/docs/transformers/model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index c2e6933d44d8..335d42d7b900 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -337,6 +337,8 @@ title: MarkupLM - local: model_doc/mbart title: MBart and MBart-50 + - local: model_doc/mega + title: MEGA - local: model_doc/megatron-bert title: MegatronBERT - local: model_doc/megatron_gpt2 diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index eee27503451c..f67483f84f03 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -151,6 +151,7 @@ The documentation is organized into five sections: 1. **[MaskFormer](model_doc/maskformer)** (from Meta and UIUC) released with the paper [Per-Pixel Classification is Not All You Need for Semantic Segmentation](https://arxiv.org/abs/2107.06278) by Bowen Cheng, Alexander G. Schwing, Alexander Kirillov. 1. **[mBART](model_doc/mbart)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. 1. **[mBART-50](model_doc/mbart)** (from Facebook) released with the paper [Multilingual Translation with Extensible Multilingual Pretraining and Finetuning](https://arxiv.org/abs/2008.00401) by Yuqing Tang, Chau Tran, Xian Li, Peng-Jen Chen, Naman Goyal, Vishrav Chaudhary, Jiatao Gu, Angela Fan. +1. **[MEGA](model_doc/mega)** (from Facebook) released with the paper [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) by Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer. 1. **[Megatron-BERT](model_doc/megatron-bert)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[Megatron-GPT2](model_doc/megatron_gpt2)** (from NVIDIA) released with the paper [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro. 1. **[MGP-STR](model_doc/mgp-str)** (from Alibaba Research) released with the paper [Multi-Granularity Prediction for Scene Text Recognition](https://arxiv.org/abs/2209.03592) by Peng Wang, Cheng Da, and Cong Yao. @@ -345,6 +346,7 @@ Flax), PyTorch, and/or TensorFlow. | MaskFormer | ❌ | ❌ | ✅ | ❌ | ❌ | | MaskFormerSwin | ❌ | ❌ | ❌ | ❌ | ❌ | | mBART | ✅ | ✅ | ✅ | ✅ | ✅ | +| MEGA | ❌ | ❌ | ✅ | ❌ | ❌ | | Megatron-BERT | ❌ | ❌ | ✅ | ❌ | ❌ | | MGP-STR | ✅ | ❌ | ✅ | ❌ | ❌ | | MobileBERT | ✅ | ✅ | ✅ | ✅ | ❌ | diff --git a/docs/source/en/model_doc/mega.mdx b/docs/source/en/model_doc/mega.mdx new file mode 100644 index 000000000000..bfde22a5b693 --- /dev/null +++ b/docs/source/en/model_doc/mega.mdx @@ -0,0 +1,78 @@ + + +# MEGA + +## Overview + +The MEGA model was proposed in [Mega: Moving Average Equipped Gated Attention](https://arxiv.org/abs/2209.10655) by Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, Jonathan May, and Luke Zettlemoyer. +MEGA proposes a new approach to self-attention with each encoder layer having a multi-headed exponential moving average in addition to a single head of standard dot-product attention, giving the attention mechanism +stronger positional biases. This allows MEGA to perform competitively to Transformers on standard benchmarks including LRA +while also having significantly fewer parameters. MEGA's compute efficiency allows it to scale to very long sequences, making it an +attractive option for long-document NLP tasks. + +The abstract from the paper is the following: + + *The design choices in the Transformer attention mechanism, including weak inductive bias and quadratic computational complexity, have limited its application for modeling long sequences. In this paper, we introduce Mega, a simple, theoretically grounded, single-head gated attention mechanism equipped with (exponential) moving average to incorporate inductive bias of position-aware local dependencies into the position-agnostic attention mechanism. We further propose a variant of Mega that offers linear time and space complexity yet yields only minimal quality loss, by efficiently splitting the whole sequence into multiple chunks with fixed length. Extensive experiments on a wide range of sequence modeling benchmarks, including the Long Range Arena, neural machine translation, auto-regressive language modeling, and image and speech classification, show that Mega achieves significant improvements over other sequence models, including variants of Transformers and recent state space models. * + +Tips: + +- MEGA can perform quite well with relatively few parameters. See Appendix D in the MEGA paper for examples of architectural specs which perform well in various settings. If using MEGA as a decoder, be sure to set `bidirectional=False` to avoid errors with default bidirectional. +- Mega-chunk is a variant of mega that reduces time and spaces complexity from quadratic to linear. Utilize chunking with MegaConfiig.use_chunking and control chunk size with MegaConfig.chunk_size + +This model was contributed by [mnaylor](https://huggingface.co/mnaylor). +The original code can be found [here](https://github.com/facebookresearch/mega). + +Implementation Notes: + +- The original implementation of MEGA had an inconsistent expectation of attention masks for padding and causal self-attention between the softmax attention and Laplace/squared ReLU method. This implementation addresses that inconsistency. +- The original implementation did not include token type embeddings; this implementation adds support for these, with the option controlled by MegaConfig.add_token_type_embeddings + + +## MegaConfig + +[[autodoc]] MegaConfig + +## MegaModel + +[[autodoc]] MegaModel + - forward + +## MegaForCausalLM + +[[autodoc]] MegaForCausalLM + - forward + +## MegaForMaskedLM + +[[autodoc]] MegaForMaskedLM + - forward + +## MegaForSequenceClassification + +[[autodoc]] MegaForSequenceClassification + - forward + +## MegaForMultipleChoice + +[[autodoc]] MegaForMultipleChoice + - forward + +## MegaForTokenClassification + +[[autodoc]] MegaForTokenClassification + - forward + +## MegaForQuestionAnswering + +[[autodoc]] MegaForQuestionAnswering + - forward diff --git a/docs/source/en/serialization.mdx b/docs/source/en/serialization.mdx index f57ea59971d2..29fab3724d56 100644 --- a/docs/source/en/serialization.mdx +++ b/docs/source/en/serialization.mdx @@ -94,6 +94,7 @@ Ready-made configurations include the following architectures: - M2M100 - Marian - mBART +- MEGA - MobileBERT - MobileNetV1 - MobileNetV2 diff --git a/docs/source/en/tasks/language_modeling.mdx b/docs/source/en/tasks/language_modeling.mdx index d8801888d378..a5e4ae6bb38e 100644 --- a/docs/source/en/tasks/language_modeling.mdx +++ b/docs/source/en/tasks/language_modeling.mdx @@ -34,7 +34,7 @@ Choose one of the following architectures: -[BART](../model_doc/bart), [BERT](../model_doc/bert), [Bert Generation](../model_doc/bert-generation), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BioGpt](../model_doc/biogpt), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CodeGen](../model_doc/codegen), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [GIT](../model_doc/git), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT NeoX](../model_doc/gpt_neox), [GPT NeoX Japanese](../model_doc/gpt_neox_japanese), [GPT-J](../model_doc/gptj), [LLaMA](../model_doc/llama), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MVP](../model_doc/mvp), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Pegasus](../model_doc/pegasus), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [Speech2Text2](../model_doc/speech_to_text_2), [Transformer-XL](../model_doc/transfo-xl), [TrOCR](../model_doc/trocr), [XGLM](../model_doc/xglm), [XLM](../model_doc/xlm), [XLM-ProphetNet](../model_doc/xlm-prophetnet), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod) +[BART](../model_doc/bart), [BERT](../model_doc/bert), [Bert Generation](../model_doc/bert-generation), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BioGpt](../model_doc/biogpt), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CodeGen](../model_doc/codegen), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [GIT](../model_doc/git), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT NeoX](../model_doc/gpt_neox), [GPT NeoX Japanese](../model_doc/gpt_neox_japanese), [GPT-J](../model_doc/gptj), [LLaMA](../model_doc/llama), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MVP](../model_doc/mvp), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Pegasus](../model_doc/pegasus), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [Speech2Text2](../model_doc/speech_to_text_2), [Transformer-XL](../model_doc/transfo-xl), [TrOCR](../model_doc/trocr), [XGLM](../model_doc/xglm), [XLM](../model_doc/xlm), [XLM-ProphetNet](../model_doc/xlm-prophetnet), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod) diff --git a/docs/source/en/tasks/masked_language_modeling.mdx b/docs/source/en/tasks/masked_language_modeling.mdx index e8a69123508d..c41d6493c589 100644 --- a/docs/source/en/tasks/masked_language_modeling.mdx +++ b/docs/source/en/tasks/masked_language_modeling.mdx @@ -31,7 +31,7 @@ Choose one of the following architectures: -[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [CamemBERT](../model_doc/camembert), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [Perceiver](../model_doc/perceiver), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [TAPAS](../model_doc/tapas), [Wav2Vec2](../model_doc/wav2vec2), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) +[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [CamemBERT](../model_doc/camembert), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [mBART](../model_doc/mbart), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [Perceiver](../model_doc/perceiver), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [TAPAS](../model_doc/tapas), [Wav2Vec2](../model_doc/wav2vec2), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) diff --git a/docs/source/en/tasks/multiple_choice.mdx b/docs/source/en/tasks/multiple_choice.mdx index 874d22985964..4e4ff06e7f71 100644 --- a/docs/source/en/tasks/multiple_choice.mdx +++ b/docs/source/en/tasks/multiple_choice.mdx @@ -26,7 +26,7 @@ The task illustrated in this tutorial is supported by the following model archit -[ALBERT](../model_doc/albert), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [I-BERT](../model_doc/ibert), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [QDQBert](../model_doc/qdqbert), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) +[ALBERT](../model_doc/albert), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [I-BERT](../model_doc/ibert), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [QDQBert](../model_doc/qdqbert), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) diff --git a/docs/source/en/tasks/question_answering.mdx b/docs/source/en/tasks/question_answering.mdx index 9f6ebdc5d002..c1f6a7ccd2f6 100644 --- a/docs/source/en/tasks/question_answering.mdx +++ b/docs/source/en/tasks/question_answering.mdx @@ -31,7 +31,7 @@ The task illustrated in this tutorial is supported by the following model archit -[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-J](../model_doc/gptj), [I-BERT](../model_doc/ibert), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LED](../model_doc/led), [LiLT](../model_doc/lilt), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [LXMERT](../model_doc/lxmert), [MarkupLM](../model_doc/markuplm), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [OPT](../model_doc/opt), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [Splinter](../model_doc/splinter), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) +[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-J](../model_doc/gptj), [I-BERT](../model_doc/ibert), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LED](../model_doc/led), [LiLT](../model_doc/lilt), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [LXMERT](../model_doc/lxmert), [MarkupLM](../model_doc/markuplm), [mBART](../model_doc/mbart), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [OPT](../model_doc/opt), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [Splinter](../model_doc/splinter), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) diff --git a/docs/source/en/tasks/sequence_classification.mdx b/docs/source/en/tasks/sequence_classification.mdx index b4c57b3fa6f8..29d88dbe2f01 100644 --- a/docs/source/en/tasks/sequence_classification.mdx +++ b/docs/source/en/tasks/sequence_classification.mdx @@ -28,7 +28,7 @@ The task illustrated in this tutorial is supported by the following model archit -[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT-J](../model_doc/gptj), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LED](../model_doc/led), [LiLT](../model_doc/lilt), [LLaMA](../model_doc/llama), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [mBART](../model_doc/mbart), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Perceiver](../model_doc/perceiver), [PLBart](../model_doc/plbart), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [TAPAS](../model_doc/tapas), [Transformer-XL](../model_doc/transfo-xl), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) +[ALBERT](../model_doc/albert), [BART](../model_doc/bart), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [CTRL](../model_doc/ctrl), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [GPT Neo](../model_doc/gpt_neo), [GPT-J](../model_doc/gptj), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LED](../model_doc/led), [LiLT](../model_doc/lilt), [LLaMA](../model_doc/llama), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [mBART](../model_doc/mbart), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [MVP](../model_doc/mvp), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [OpenAI GPT](../model_doc/openai-gpt), [OPT](../model_doc/opt), [Perceiver](../model_doc/perceiver), [PLBart](../model_doc/plbart), [QDQBert](../model_doc/qdqbert), [Reformer](../model_doc/reformer), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [TAPAS](../model_doc/tapas), [Transformer-XL](../model_doc/transfo-xl), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) diff --git a/docs/source/en/tasks/token_classification.mdx b/docs/source/en/tasks/token_classification.mdx index 7f6032a0ec4e..930361a354cd 100644 --- a/docs/source/en/tasks/token_classification.mdx +++ b/docs/source/en/tasks/token_classification.mdx @@ -28,7 +28,7 @@ The task illustrated in this tutorial is supported by the following model archit -[ALBERT](../model_doc/albert), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LiLT](../model_doc/lilt), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [QDQBert](../model_doc/qdqbert), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) +[ALBERT](../model_doc/albert), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LiLT](../model_doc/lilt), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [QDQBert](../model_doc/qdqbert), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 1be1a6f74f9f..bc32232d51d2 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -370,6 +370,7 @@ "models.mbart": ["MBartConfig"], "models.mbart50": [], "models.mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig", "MCTCTProcessor"], + "models.mega": ["MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegaConfig"], "models.megatron_bert": ["MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegatronBertConfig"], "models.megatron_gpt2": [], "models.mgp_str": ["MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP", "MgpstrConfig", "MgpstrProcessor", "MgpstrTokenizer"], @@ -1913,6 +1914,19 @@ "MCTCTPreTrainedModel", ] ) + _import_structure["models.mega"].extend( + [ + "MEGA_PRETRAINED_MODEL_ARCHIVE_LIST", + "MegaForCausalLM", + "MegaForMaskedLM", + "MegaForMultipleChoice", + "MegaForQuestionAnswering", + "MegaForSequenceClassification", + "MegaForTokenClassification", + "MegaModel", + "MegaPreTrainedModel", + ] + ) _import_structure["models.megatron_bert"].extend( [ "MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -4005,6 +4019,7 @@ from .models.maskformer import MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, MaskFormerConfig, MaskFormerSwinConfig from .models.mbart import MBartConfig from .models.mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig, MCTCTProcessor + from .models.mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig from .models.megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig from .models.mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig, MgpstrProcessor, MgpstrTokenizer from .models.mmbt import MMBTConfig @@ -5298,6 +5313,17 @@ MBartPreTrainedModel, ) from .models.mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel + from .models.mega import ( + MEGA_PRETRAINED_MODEL_ARCHIVE_LIST, + MegaForCausalLM, + MegaForMaskedLM, + MegaForMultipleChoice, + MegaForQuestionAnswering, + MegaForSequenceClassification, + MegaForTokenClassification, + MegaModel, + MegaPreTrainedModel, + ) from .models.megatron_bert import ( MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST, MegatronBertForCausalLM, diff --git a/src/transformers/activations.py b/src/transformers/activations.py index 436d2b95fe62..587dc2e59964 100644 --- a/src/transformers/activations.py +++ b/src/transformers/activations.py @@ -121,6 +121,22 @@ def forward(self, x: Tensor) -> Tensor: return torch.clip(gelu(x), self.min, self.max) +class AccurateGELUActivation(nn.Module): + """ + Applies GELU approximation that is faster than default and more accurate than QuickGELU. See: + https://github.com/hendrycks/GELUs + + Implemented along with MEGA (Moving Average Equipped Gated Attention) + """ + + def __init__(self): + super().__init__() + self.precomputed_constant = math.sqrt(2 / math.pi) + + def forward(self, input: Tensor) -> Tensor: + return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044715 * torch.pow(input, 3)))) + + class SiLUActivation(nn.Module): """ See Gaussian Error Linear Units (Hendrycks et al., https://arxiv.org/abs/1606.08415) where the SiLU (Sigmoid Linear @@ -163,6 +179,30 @@ def forward(self, input: Tensor) -> Tensor: return input +class LaplaceActivation(nn.Module): + """ + Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See + https://arxiv.org/abs/2209.10655 + + Inspired by squared relu, but with bounded range and gradient for better stability + """ + + def forward(self, input, mu=0.707107, sigma=0.282095): + input = (input - mu).div(sigma * math.sqrt(2.0)) + return 0.5 * (1.0 + torch.erf(input)) + + +class ReLUSquaredActivation(nn.Module): + """ + Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2 + """ + + def forward(self, input): + relu_applied = nn.functional.relu(input) + squared = torch.square(relu_applied) + return squared + + class ClassInstantier(OrderedDict): def __getitem__(self, key): content = super().__getitem__(key) @@ -177,10 +217,13 @@ def __getitem__(self, key): "gelu_new": NewGELUActivation, "gelu_python": (GELUActivation, {"use_gelu_python": True}), "gelu_pytorch_tanh": PytorchGELUTanh, + "gelu_accurate": AccurateGELUActivation, + "laplace": LaplaceActivation, "linear": LinearActivation, "mish": MishActivation, "quick_gelu": QuickGELUActivation, "relu": nn.ReLU, + "relu2": ReLUSquaredActivation, "relu6": nn.ReLU6, "sigmoid": nn.Sigmoid, "silu": SiLUActivation, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index d32aa2f6e884..e6e936148b61 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -116,6 +116,7 @@ mbart, mbart50, mctct, + mega, megatron_bert, megatron_gpt2, mgp_str, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 9a8500d118ff..32c5da7bd143 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -122,6 +122,7 @@ ("maskformer-swin", "MaskFormerSwinConfig"), ("mbart", "MBartConfig"), ("mctct", "MCTCTConfig"), + ("mega", "MegaConfig"), ("megatron-bert", "MegatronBertConfig"), ("mgp-str", "MgpstrConfig"), ("mobilebert", "MobileBertConfig"), @@ -299,6 +300,7 @@ ("maskformer", "MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mbart", "MBART_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mctct", "MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("mega", "MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("megatron-bert", "MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mgp-str", "MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mobilenet_v1", "MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -484,6 +486,7 @@ ("mbart", "mBART"), ("mbart50", "mBART-50"), ("mctct", "M-CTC-T"), + ("mega", "MEGA"), ("megatron-bert", "Megatron-BERT"), ("megatron_gpt2", "Megatron-GPT2"), ("mgp-str", "MGP-STR"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index eff11b45a53f..e8f295d1c12d 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -120,6 +120,7 @@ ("maskformer-swin", "MaskFormerSwinModel"), ("mbart", "MBartModel"), ("mctct", "MCTCTModel"), + ("mega", "MegaModel"), ("megatron-bert", "MegatronBertModel"), ("mgp-str", "MgpstrForSceneTextRecognition"), ("mobilebert", "MobileBertModel"), @@ -228,6 +229,7 @@ ("longformer", "LongformerForMaskedLM"), ("luke", "LukeForMaskedLM"), ("lxmert", "LxmertForPreTraining"), + ("mega", "MegaForMaskedLM"), ("megatron-bert", "MegatronBertForPreTraining"), ("mobilebert", "MobileBertForPreTraining"), ("mpnet", "MPNetForMaskedLM"), @@ -302,6 +304,7 @@ ("luke", "LukeForMaskedLM"), ("m2m_100", "M2M100ForConditionalGeneration"), ("marian", "MarianMTModel"), + ("mega", "MegaForMaskedLM"), ("megatron-bert", "MegatronBertForCausalLM"), ("mobilebert", "MobileBertForMaskedLM"), ("mpnet", "MPNetForMaskedLM"), @@ -363,6 +366,7 @@ ("llama", "LlamaForCausalLM"), ("marian", "MarianForCausalLM"), ("mbart", "MBartForCausalLM"), + ("mega", "MegaForCausalLM"), ("megatron-bert", "MegatronBertForCausalLM"), ("mvp", "MvpForCausalLM"), ("openai-gpt", "OpenAIGPTLMHeadModel"), @@ -531,6 +535,7 @@ ("longformer", "LongformerForMaskedLM"), ("luke", "LukeForMaskedLM"), ("mbart", "MBartForConditionalGeneration"), + ("mega", "MegaForMaskedLM"), ("megatron-bert", "MegatronBertForMaskedLM"), ("mobilebert", "MobileBertForMaskedLM"), ("mpnet", "MPNetForMaskedLM"), @@ -657,6 +662,7 @@ ("luke", "LukeForSequenceClassification"), ("markuplm", "MarkupLMForSequenceClassification"), ("mbart", "MBartForSequenceClassification"), + ("mega", "MegaForSequenceClassification"), ("megatron-bert", "MegatronBertForSequenceClassification"), ("mobilebert", "MobileBertForSequenceClassification"), ("mpnet", "MPNetForSequenceClassification"), @@ -719,6 +725,7 @@ ("lxmert", "LxmertForQuestionAnswering"), ("markuplm", "MarkupLMForQuestionAnswering"), ("mbart", "MBartForQuestionAnswering"), + ("mega", "MegaForQuestionAnswering"), ("megatron-bert", "MegatronBertForQuestionAnswering"), ("mobilebert", "MobileBertForQuestionAnswering"), ("mpnet", "MPNetForQuestionAnswering"), @@ -796,6 +803,7 @@ ("longformer", "LongformerForTokenClassification"), ("luke", "LukeForTokenClassification"), ("markuplm", "MarkupLMForTokenClassification"), + ("mega", "MegaForTokenClassification"), ("megatron-bert", "MegatronBertForTokenClassification"), ("mobilebert", "MobileBertForTokenClassification"), ("mpnet", "MPNetForTokenClassification"), @@ -838,6 +846,7 @@ ("ibert", "IBertForMultipleChoice"), ("longformer", "LongformerForMultipleChoice"), ("luke", "LukeForMultipleChoice"), + ("mega", "MegaForMultipleChoice"), ("megatron-bert", "MegatronBertForMultipleChoice"), ("mobilebert", "MobileBertForMultipleChoice"), ("mpnet", "MPNetForMultipleChoice"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 5afedadca446..900c0b836d8e 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -194,6 +194,7 @@ "MBart50TokenizerFast" if is_tokenizers_available() else None, ), ), + ("mega", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ("megatron-bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), ("mgp-str", ("MgpstrTokenizer", None)), ("mluke", ("MLukeTokenizer" if is_sentencepiece_available() else None, None)), diff --git a/src/transformers/models/mega/__init__.py b/src/transformers/models/mega/__init__.py new file mode 100644 index 000000000000..728499ef2d38 --- /dev/null +++ b/src/transformers/models/mega/__init__.py @@ -0,0 +1,70 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from ...utils import ( + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, +) + + +_import_structure = { + "configuration_mega": ["MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegaConfig", "MegaOnnxConfig"], +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_mega"] = [ + "MEGA_PRETRAINED_MODEL_ARCHIVE_LIST", + "MegaForCausalLM", + "MegaForMaskedLM", + "MegaForMultipleChoice", + "MegaForQuestionAnswering", + "MegaForSequenceClassification", + "MegaForTokenClassification", + "MegaModel", + "MegaPreTrainedModel", + ] + +if TYPE_CHECKING: + from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_mega import ( + MEGA_PRETRAINED_MODEL_ARCHIVE_LIST, + MegaForCausalLM, + MegaForMaskedLM, + MegaForMultipleChoice, + MegaForQuestionAnswering, + MegaForSequenceClassification, + MegaForTokenClassification, + MegaModel, + MegaPreTrainedModel, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/mega/configuration_mega.py b/src/transformers/models/mega/configuration_mega.py new file mode 100644 index 000000000000..cade307c84e5 --- /dev/null +++ b/src/transformers/models/mega/configuration_mega.py @@ -0,0 +1,242 @@ +# coding=utf-8 +# Copyright 2023 The Mega Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MEGA configuration""" +from collections import OrderedDict +from typing import Mapping + +from ...configuration_utils import PretrainedConfig +from ...onnx import OnnxConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "mnaylor/mega-base-wikitext": "https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json", +} + + +class MegaConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`MegaModel`]. It is used to instantiate a Mega + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the Mega + [mnaylor/mega-base-wikitext](https://huggingface.co/mnaylor/mega-base-wikitext) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 30522): + Vocabulary size of the Mega model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`MegaModel`]. + hidden_size (`int`, *optional*, defaults to 128): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (`int`, *optional*, defaults to 4): + Number of hidden layers in the Mega encoder. + intermediate_size (`int`, *optional*, defaults to 256): + Dimensionality of the hidden size (self-attention value projection) within the Mega encoder + ema_projection_size (`int`, *optional*, defaults to 16): + Dimensionality of the MegaMultiDimensionDampedEma + bidirectional (`bool`, *optional*, defaults to `True`): + Whether the MegaMultiDimensionDampedEma used in Mega's self-attention should work bidirectionally (`True`) + or unidirectionally (`False`). Bidirectional EMA is incompatible with causal decoding, so this should be + False if you intend to use the model as a decoder. + shared_representation_size (`int`, *optional*, defaults to 64): + Dimensionality of the linear projection for shared representation of self-attention queries and keys + use_chunking (`bool`, *optional*, defaults to `False`): + Whether to chunk inputs for linear self-attention complexity (described as Mega-chunk in the paper) + chunk_size (`int`, *optional*, defaults to -1): + If `use_chunking` is set to `True`, determines the size of the chunks to apply to the input sequence. If + chunking is used, input sequences must be padded to a multiple of `chunk_size` + truncation (`int`, *optional*): + If specified, the sequence length for which to truncate MegaMultiDimensionDampedEma + normalize_before_mega (`bool`, *optional*, defaults to `True`): + Whether to normalize before (`True`) or after (`False`) passing through Mega encoder blocks + normalization_type (`str`, *optional*, defaults to `"scalenorm"`): + Type of normalization to use in Mega encoder blocks. Choose one of `"scalenorm"`, `"layernorm"`, + `"rmsnorm"`, `"batchnorm"`, or `"syncbatchnorm"` (GPU required for syncbatchnorm) + norm_affine (`bool`, *optional*, defaults to `True`): + If `True`, applies a parameterized affine transformation to inputs during normalization + activation (`str`, *optional*, defaults to `"silu"`): + Activation function to apply within Mega encoder blocks. Choose one of `"silu"`, `"relu"`, `"linear"`, + `"gelu"`, or `"gelu_accurate"` + attention_activation (`str`, *optional*, defaults to `"softmax"`): + Activation function to apply for single-headed self-attention (a la Transformer). Choose one of + `"softmax"`, `"laplace"`, or `"relu2"` + dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for EMA self-attention + hidden_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention probabilities. + use_feature_dropout (`bool`, *optional*, defaults to `False`): + Whether to use feature-based (`True`) or standard dropout (`False`) + use_normalized_ffn (`bool`, *optional*, defaults to `True`): + Whether to use the normalized feed-forward sub-layer in Mega blocks (`True`) or pass Mega encoder output + as-is (`False`) + nffn_hidden_size (`int`, *optional*, defaults to 256): + If using the normalized feed-forward network (NFFN) layer within Mega (`use_normalized_ffn = True`), this + is the hidden size of the NFFN + normalize_before_ffn (`bool`, *optional*, defaults to `True`): + Whether to normalize before (`True`) or after (`False`) the feed-forward portion of NFFN + nffn_activation_dropout_prob (`float`, *optional*, defaults to 0.1): + The dropout ratio for the NFFN component. + max_positions (`int`, *optional*, defaults to 2048): + The maximum sequence length to use for positional representations. For `"simple"` relative positional bias, + this is a hard limit on input length; `"rotary"` relative positional bias will extrapolate to longer + sequences + add_token_type_embeddings (`bool`, *optional*, defaults to `True`): + Whether to account for token types in embeddings. Left as optional to maintain compatibility with original + implementation while adding support for token types. + type_vocab_size (`int`, *optional*, defaults to 2): + The vocabulary size of the `token_type_ids` passed when calling [`MegaModel`]. Only used if + `add_token_type_embeddings = True` + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + ema_delta_alpha_range (`float`, *optional*, defaults to 0.2): + The standard deviation for initializing the delta (damping factor) and alpha (decay factor) parameters in + MegaMultiDimensionDampedEma. + ema_beta_range (`float`, *optional*, defaults to 0.02): + The standard deviation for initializing the beta parameter (expansion matrix) in + MegaMultiDimensionDampedEma. + ema_gamma_omega_range (`float`, *optional*, defaults to 1.0): + The standard deviation for initializing the gamma (projection matrix) and omega (residual weight) + parameters in MultiDimensionEMA. + relative_positional_bias (`str`, *optional*, defaults to `"rotary"`): + Type of relative positional encoding. Choose one of `"rotary"` or `"simple"`. If `"simple"` is selected, + `max_positions` is used as a limit on input size, while `"rotary"` extrapolates beyond `max_positions`. + is_decoder (`bool`, *optional*, defaults to `False`): + Whether the model is used as a decoder or not. If `False`, the model is used as an encoder. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + classifier_dropout (`float`, *optional*): + The dropout ratio for the classification head. + add_lm_hidden_dense_layer (`bool`, *optional*, defaults to `True`): + Whether to include a hidden layer for projection between encoder outputs and LM heads (`True`) or pass + hidden states directly to LM head (`False`). Remains optional for compatibility with original + implementation + + Examples: + + ```python + >>> from transformers import MegaConfig, MegaModel + + >>> # Initializing a Mega configuration + >>> configuration = MegaConfig() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = MegaModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "mega" + + def __init__( + self, + vocab_size=30522, + hidden_size=128, + num_hidden_layers=4, + intermediate_size=256, + ema_projection_size=16, + bidirectional=True, + shared_representation_size=64, + use_chunking=False, + chunk_size=-1, + truncation=None, + normalize_before_mega=True, + normalization_type="scalenorm", + norm_affine=True, + activation="silu", + attention_activation="softmax", + dropout_prob=0.1, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + use_feature_dropout=False, + use_normalized_ffn=True, + nffn_hidden_size=256, + normalize_before_ffn=True, + nffn_activation_dropout_prob=0.1, + max_positions=2048, + add_token_type_embeddings=False, + type_vocab_size=2, + initializer_range=0.02, + ema_delta_alpha_range=0.2, + ema_beta_range=0.02, + ema_gamma_omega_range=1.0, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + relative_positional_bias="rotary", + classifier_dropout=None, + use_cache=True, + add_lm_hidden_dense_layer=True, + **kwargs, + ): + super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.activation = activation + self.attention_activation = attention_activation + self.intermediate_size = intermediate_size + self.ema_projection_size = ema_projection_size + self.bidirectional = bidirectional + self.shared_representation_size = shared_representation_size + self.use_chunking = use_chunking + self.chunk_size = chunk_size + self.truncation = truncation + self.normalize_before_mega = normalize_before_mega + self.normalization_type = normalization_type + self.norm_affine = norm_affine + self.dropout_prob = dropout_prob + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.use_feature_dropout = use_feature_dropout + self.use_normalized_ffn = use_normalized_ffn + self.nffn_hidden_size = nffn_hidden_size + self.normalize_before_ffn = normalize_before_ffn + self.nffn_activation_dropout_prob = nffn_activation_dropout_prob + self.max_positions = max_positions + self.add_token_type_embeddings = add_token_type_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.ema_delta_alpha_range = ema_delta_alpha_range + self.ema_beta_range = ema_beta_range + self.ema_gamma_omega_range = ema_gamma_omega_range + self.relative_positional_bias = relative_positional_bias + self.use_cache = use_cache + self.classifier_dropout = classifier_dropout + self.add_lm_hidden_dense_layer = add_lm_hidden_dense_layer + self.num_attention_heads = 1 # not used but required by Hugging Face + + +class MegaOnnxConfig(OnnxConfig): + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + if self.task == "multiple-choice": + dynamic_axis = {0: "batch", 1: "choice", 2: "sequence"} + else: + dynamic_axis = {0: "batch", 1: "sequence"} + return OrderedDict( + [ + ("input_ids", dynamic_axis), + ("attention_mask", dynamic_axis), + ] + ) diff --git a/src/transformers/models/mega/convert_mega_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/mega/convert_mega_original_pytorch_checkpoint_to_pytorch.py new file mode 100644 index 000000000000..2fe75ba27324 --- /dev/null +++ b/src/transformers/models/mega/convert_mega_original_pytorch_checkpoint_to_pytorch.py @@ -0,0 +1,291 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Convert Mega pretrained checkpoint. Built to convert the Masked LM checkpoint located at +https://huggingface.co/mnaylor/mega-wikitext-103 + +Requirements: + - clone the Mega repo and install fairseq from there + 1. git clone https://github.com/facebookresearch/mega.git + 2. cd mega && pip install -e + - clone the pretrained weights for the original implementation from the hugging face repo + * use this location as the path for pretrained weights +""" +import argparse + +# utilities to import the model weights and config file +import os +import pickle as pkl + +# PyTorch + new model classes +import torch +from torch import nn + +from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM + + +# import the EncoderLayer class used to pretrain +# !! NOTE !! this requires the version of fairseq that is built when you install the Mega source +try: + from fairseq.modules.mega_layer import MegaEncoderLayer +except ImportError: + raise ImportError("You need to install the version of fairseq from the Mega repo!") + + +# define the wrapper classes used to train the MLM (see colab notebook below) +# https://colab.research.google.com/drive/1qfUO6o5HRdxBblWlw058HVyvaEPhPpH8?usp=sharing +# MegaLM outputs hidden states +class MegaLM(nn.Module): + "The base class for our Mega encoder - given input IDs, embed text and return encoder output" + + def __init__(self, mega_args, depth, vocab_size): + super().__init__() + self.mega_args = mega_args + self.embedding_layer = nn.Embedding(vocab_size, self.mega_args.encoder_embed_dim) + self.encoders = nn.ModuleList([MegaEncoderLayer(self.mega_args) for _ in range(depth)]) + self.depth = depth + + def forward(self, input_ids, attention_mask, batch_first=True, ignore_mask_value=0): + """ + Code for a forward pass - expects input_ids and attention_mask to come from a Hugging Face tokenizer as PyTorch + tensors, and returns a tensor of size (batch, n_classes) containing classification logits + + Other options: + - batch_first: boolean indicating whether the batch dimension is first in input_ids (default: True, which + aligns with the HF tokenizer behavior) + - ignore_mask_value: the value in attention_mask that identifies tokens that should be ignored (default: 0, + which aligns with HF tokenizer) + """ + + # Mega expects embeddings to be (time, batch, embedding size), but + # Hugging Face returns tokens as (batch, time) + if batch_first: + input_ids = input_ids.T + + # to make things more confusing, Mega expects the attention mask to + # be (batch, time), but with values of 0 (normal token) and 1 (ignore token) + # which is the opposite of what HF returns + if ignore_mask_value == 0: + attention_mask = 1 - attention_mask + + # get token embeddings from IDs + embeds = self.embedding_layer(input_ids) + + # pass through the Mega layers + # input is (time, batch, encoder dim) and output is the same + for encoder in self.encoders: + embeds = encoder(embeds, attention_mask) + + # return according to the shape specified + if batch_first: + # (T, B, H) --> (B, T, H) + return torch.transpose(embeds, 0, 1) + else: + return embeds + + +# renamed from MegaForMaskedLM to avoid confusion with new module +class OriginalMegaForMaskedLM(nn.Module): + "A wrapper class for doing masked language modeling with Mega" + + def __init__(self, mega_args, depth, vocab_size): + super().__init__() + self.mega = MegaLM(mega_args, depth, vocab_size) + self.mlm_head = nn.Linear(mega_args.encoder_embed_dim, vocab_size) + self.dropout = nn.Dropout(p=0.1) + + def forward(self, input_ids, attention_mask, batch_first=True, ignore_mask_value=0): + """ + Perform a forward pass through the Mega encoder and the masked LM head. Returns logits for each vocabulary + entry. + + If `batch_first` (default to align with Hugging Face tokenizer behavior), output will have the shape (Batch + size, Sequence length, Vocab size); otherwise (S, B, V) + """ + encoder_output = self.mega(input_ids, attention_mask, batch_first, ignore_mask_value) + return self.mlm_head(self.dropout(encoder_output)) + + +# code to convert the checkpoint located in the user-specified location +def convert_checkpoint_to_huggingface(pretrained_checkpoint_path, output_path, includes_tokenizer): + with open(os.path.join(pretrained_checkpoint_path, "model_args.pkl"), "rb") as f: + mega_original_args = pkl.load(f) + + # load the original encoder + original_mlm = OriginalMegaForMaskedLM(**mega_original_args).eval() + + # load its weights + print( + "Original Mega encoder:", + original_mlm.mega.load_state_dict( + torch.load(os.path.join(pretrained_checkpoint_path, "encoder_weights.pt"), map_location="cpu") + ), + ) + print( + "Original Mega MLM layer:", + original_mlm.mlm_head.load_state_dict( + torch.load(os.path.join(pretrained_checkpoint_path, "mlm_head_weights.pt"), map_location="cpu") + ), + ) + + # create a new config from the old one + hf_config = MegaConfig( + num_hidden_layers=mega_original_args["depth"], + vocab_size=mega_original_args["vocab_size"], + hidden_size=mega_original_args["mega_args"].encoder_embed_dim, + shared_representation_size=mega_original_args["mega_args"].encoder_z_dim, + intermediate_size=mega_original_args["mega_args"].encoder_hidden_dim, + ema_projection_size=mega_original_args["mega_args"].encoder_n_dim, + dropout_prob=mega_original_args["mega_args"].dropout, + attention_probs_dropout_prob=mega_original_args["mega_args"].attention_dropout, + hidden_dropout_prob=mega_original_args["mega_args"].hidden_dropout, + activation=mega_original_args["mega_args"].activation_fn, + attention_activation=mega_original_args["mega_args"].attention_activation_fn, + bidirectional=mega_original_args["mega_args"].bidirectional, + use_chunking=mega_original_args["mega_args"].encoder_chunk_size > 0, + chunk_size=mega_original_args["mega_args"].encoder_chunk_size, + truncation=mega_original_args["mega_args"].truncation_length, + normalization_type=mega_original_args["mega_args"].normalization_type, + normalize_before_mega=True, + norm_affine=True, + use_feature_dropout=mega_original_args["mega_args"].feature_dropout, + relative_positional_bias=mega_original_args["mega_args"].rel_pos_bias, + max_positions=mega_original_args["mega_args"].max_source_positions, + nffn_hidden_size=mega_original_args["mega_args"].encoder_ffn_embed_dim, + normalize_before_ffn=mega_original_args["mega_args"].normalize_before, + # new arguments added for HF implementation + nffn_activation_dropout_prob=0.0, + add_token_type_embeddings=False, + add_lm_hidden_dense_layer=False, + ) + + hf_mlm = MegaForMaskedLM(hf_config).eval() + + # the originl checkpoint just uses nn.Embedding for the word embeddings + # we use a wrapper module for embeddings to add support for positional embeddings + hf_mlm.mega.embedding_layer.word_embeddings.weight = original_mlm.mega.embedding_layer.weight + + # modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face + # ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained, + # also renaming previously confusing parameter names + original_state_dict = original_mlm.mega.encoders.state_dict() + updated_keys = {} + for module_name in original_state_dict.keys(): + new_module_name = None + # have to handle gamma, beta, and alpha differently due to their use + # in multiple modules within the original repository; + # beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights + # the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here + if "beta" in module_name: + # EMA sub-layers were always called "move" in the original repo + if "move.beta" in module_name: + new_module_name = module_name.replace("move.beta", "ema_gate.ema_expansion_matrix") + elif "mega_layer.beta" in module_name: + new_module_name = module_name.replace("beta", "qk_bias") + else: + new_module_name = module_name.replace("beta", "b_param") + # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights + elif "gamma" in module_name: + if "move.gamma" in module_name: + new_module_name = module_name.replace("move.gamma", "ema_gate.kernel_projection_matrix") + elif "mega_layer.gamma" in module_name: + new_module_name = module_name.replace("gamma", "qk_weight") + else: + new_module_name = module_name.replace("gamma", "g_param") + # alpha is used in EMA and positional bias; renaming to improve readability + elif "move.alpha" in module_name: + new_module_name = module_name.replace("move.alpha", "ema_gate.decay_factor") + # delta is only used in EMA; renaming to improve readability + elif "move.delta" in module_name: + new_module_name = module_name.replace("move.delta", "ema_gate.damping_factor") + # omega is only used in EMA; renaming to improve readability + elif "omega" in module_name: + new_module_name = module_name.replace("move.omega", "ema_gate.residual_weight") + + if new_module_name: + updated_keys[module_name] = new_module_name + + if len(updated_keys) != 0: + print(f"Renaming these keys: {updated_keys.keys()}") + else: + print("No need to rename state dict entries") + for old, new in updated_keys.items(): + original_state_dict[new] = original_state_dict.pop(old) + + # now attempt to load the state dictionary with updated names + # note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style + print("HF Mega encoder:", hf_mlm.mega.layers.load_state_dict(original_state_dict)) + + # load the MLM head weights directly + print( + "HF Mega MLM layer:", + hf_mlm.mlm_head.load_state_dict( + torch.load(os.path.join(pretrained_checkpoint_path, "mlm_head_weights.pt"), map_location="cpu") + ), + ) + + # test on a randomly generated input sequence + input_ids = torch.randint(0, hf_config.vocab_size, size=(4, 256)) + input_mask = torch.ones_like(input_ids) + # mask a few tokens to make sure masking is applied appropriately :) + input_mask[:, -10:] = 0 + + # run forward passes + original_output = original_mlm(input_ids, input_mask, batch_first=True, ignore_mask_value=0) + hf_output = hf_mlm(input_ids, input_mask)[0] + + # print shapes and diff + print(f"original output {original_output.shape}") + print(f"hf output {hf_output.shape}") + print(f"max diff: {(original_output - hf_output).max()}") # 0.0 + success = torch.allclose(original_output, hf_output, atol=1e-3) + + if success: + print("Yay!") + hf_mlm.save_pretrained(output_path) + else: + raise RuntimeError(f"Something's broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}") + + if includes_tokenizer: + print("Transferring tokenizer") + tokenizer = AutoTokenizer.from_pretrained(pretrained_checkpoint_path) + tokenizer.save_pretrained(output_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--pretrained_checkpoint_path", + default=None, + type=str, + required=True, + help="Point to the directory containing your model weights using the official Mega repo", + ) + + parser.add_argument( + "--output_path", default=None, type=str, required=True, help="Location to save the Hugging Face version" + ) + + parser.add_argument( + "--includes_tokenizer", + action="store_true", + help="Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo", + ) + + args = parser.parse_args() + + convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer) diff --git a/src/transformers/models/mega/modeling_mega.py b/src/transformers/models/mega/modeling_mega.py new file mode 100644 index 000000000000..3d6b3ee9cd6c --- /dev/null +++ b/src/transformers/models/mega/modeling_mega.py @@ -0,0 +1,2300 @@ +# coding=utf-8 +# Copyright 2023 The Mega Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch MEGA model.""" + +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from ...activations import ACT2FN +from ...modeling_outputs import ( + BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions, + MaskedLMOutput, + MultipleChoiceModelOutput, + QuestionAnsweringModelOutput, + SequenceClassifierOutput, + TokenClassifierOutput, +) +from ...modeling_utils import PreTrainedModel +from ...pytorch_utils import ALL_LAYERNORM_LAYERS +from ...utils import ( + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_mega import MegaConfig + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "mnaylor/mega-base-wikitext" +_CONFIG_FOR_DOC = "MegaConfig" + +MEGA_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "mnaylor/mega-base-wikitext", + # See all Mega models at https://huggingface.co/models?filter=mega +] + + +class MegaEmbeddings(nn.Module): + """ + Mega's basic implementation does not incorporate token type embeddings, so this is a stripped-down version of + RoBERTa's embeddings which optionally includes token types + """ + + def __init__(self, config: MegaConfig): + super().__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id) + self.use_token_types = config.add_token_type_embeddings + if self.use_token_types: + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) + # registering a buffer here allows model tracing when not passing optional token type IDs + # more info at transformers issue #5664 + self.register_buffer( + "token_type_ids", torch.zeros(config.max_positions, dtype=torch.long).expand((1, -1)), persistent=False + ) + + self.padding_idx = config.pad_token_id + + def forward(self, input_ids=None, token_type_ids=None, inputs_embeds=None): + if (input_ids is None) and (inputs_embeds is None): + raise ValueError("Must provide one of input_ids or inputs_embeds") + elif input_ids is not None: + input_shape = input_ids.size() + device = input_ids.device + + # get the word embeddings if only IDs are provided + inputs_embeds = self.word_embeddings(input_ids) + else: + input_shape = inputs_embeds.size()[:-1] + device = inputs_embeds.device + + # the original Mega implementation did not include token type embeddings, so we add + # an option to use them if desired; if embeddings are present and token type IDs are + # not provided, we will use a registered buffer (which helps with tracing) + if self.use_token_types: + if token_type_ids is None: + if hasattr(self, "token_type_ids"): + buffered_token_type_ids = self.token_type_ids[:, : input_shape[1]] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand(input_shape[0], input_shape[1]) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) + + # access token type embeddings + token_type_embeddings = self.token_type_embeddings(token_type_ids) + # add the token type embeddings to the word embeddings + embeddings = inputs_embeds + token_type_embeddings + else: + embeddings = inputs_embeds + return embeddings + + +class MegaSimpleRelativePositionalBias(nn.Module): + """ + Simple relative positional embeddings copied from the Mega repo; renamed variables for better readability + """ + + def __init__(self, config: MegaConfig): + super().__init__() + self.config = config + self.max_positions = self.config.max_positions if self.config.chunk_size < 0 else self.config.chunk_size + self.rel_pos_bias = nn.Parameter(torch.Tensor(2 * config.max_positions - 1)) + + def forward(self, seq_len): + if seq_len > self.max_positions: + raise ValueError("Sequence length {} going beyond max length {}".format(seq_len, self.max_positions)) + + # seq_len * 2 - 1 + bias = self.rel_pos_bias[(self.max_positions - seq_len) : (self.max_positions + seq_len - 1)] + # seq_len * 3 - 1 + tile = F.pad(bias, (0, seq_len)) + # (seq_len * 3 - 1) * seq_len + tile = torch.tile(tile, (seq_len,)) + tile = tile[:-seq_len] + # seq_len x (3 * seq_len - 2) + tile = tile.view(seq_len, 3 * seq_len - 2) + start = (2 * seq_len - 1) // 2 + end = tile.size(1) - start + tile = tile[:, start:end] + return tile + + +class MegaRotaryRelativePositionalBias(nn.Module): + """ + Rotary relative bias for positional information; similar in concept to RoPE (i.e. RoFormer) but taken from the Mega + repo due to differences in implementation. + + When initialized, produces a positional bias which ranges from position 0 to config.max_positions, but can + extrapolate to longer sequences. Can be indexed according to input position IDs + """ + + def __init__(self, config: MegaConfig): + super().__init__() + if config.hidden_size % 2 != 0: + raise RuntimeError("Rotary positional bias requires `hidden_size` to be a multiple of 2") + self.config = config + self.embed_dim = config.shared_representation_size + self.max_positions = self.config.max_positions if self.config.chunk_size < 0 else self.config.chunk_size + self.sine, self.cosine = MegaRotaryRelativePositionalBias.get_sinusoid_embeddings( + config.max_positions, self.embed_dim + ) + # alpha and beta parameters for the rotary bias; beta renamed to b_param to avoid clashes with tf/flax weight handling + # in loading pretrained weights + self.alpha = nn.Parameter(torch.Tensor(1, self.embed_dim)) + self.b_param = nn.Parameter(torch.Tensor(1, self.embed_dim)) + self.register_buffer("_float_tensor", torch.FloatTensor([0.0])) + + @staticmethod + def get_sinusoid_embeddings(max_positions: int, embedding_dim: int): + half_dim = embedding_dim // 2 + emb = math.log(10000) / half_dim + emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb) + emb = torch.arange(max_positions, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0) + return torch.sin(emb), torch.cos(emb) + + def rotary(self, input): + seq_len, embed_dim = input.size() + chunk_1, chunk_2 = torch.chunk(input, 2, dim=-1) + if self.sine is None or seq_len > self.sine.size(0): + self.sine, self.cosine = MegaRotaryRelativePositionalBias.get_sinusoid_embeddings(seq_len, embed_dim) + self.max_positions = seq_len + self.sine = self.sine.to(self._float_tensor) + self.cosine = self.cosine.to(self._float_tensor) + + sin = self.sine[:seq_len] + cos = self.cosine[:seq_len] + return torch.cat([chunk_1 * cos - chunk_2 * sin, chunk_2 * cos + chunk_1 * sin], dim=1) + + def forward(self, seq_len): + rotary_alpha = self.rotary(self.alpha.expand(seq_len, self.embed_dim)) + rotary_beta = self.rotary(self.b_param.expand(seq_len, self.embed_dim)) + bias = torch.einsum("mk,nk->mn", rotary_alpha, rotary_beta) + return bias + + +class MegaDropout(nn.Module): + """ + A unified class for standard dropout functionality and featurewise dropout. + + The original fairseq Mega repo used 2 classes for these, which included some unnecessary handling of training logic + and an unused `inplace` option. The original implementation used torch.nn.functional instead of submodules, which + is retained here as well. + """ + + def __init__(self, dropout_probability, is_featurewise=False): + super().__init__() + self.dropout_probability = dropout_probability + self.is_featurewise = is_featurewise + + def forward(self, input, batch_first: bool = False): + if self.is_featurewise: + if batch_first: + # (batch_size X sequence_length X feature_dimension) + # -> (batch_size X feature_dimension X sequence_length) + # -> (batch_size X sequence_length X feature_dimension) + return F.dropout2d( + input.transpose(-1, -2), p=self.dropout_probability, training=self.training + ).transpose(-1, -2) + else: + if input.dim() != 3: + raise ValueError( + "Feature dropout inputs must be exactly 3-dimensional if inputs are ordered [sequence length, batch size, hidden dimension]" + ) + # (sequence_length X batch_size X feature_dimension) + # -> (batch_size X feature_dimension X sequence_length) + # -> (sequence_length X batch_size X feature_dimension) + return F.dropout2d(input.permute(1, 2, 0), p=self.dropout_probability, training=self.training).permute( + 2, 0, 1 + ) + else: + return F.dropout(input, p=self.dropout_probability, training=self.training) + + +class MegaRMSNorm(nn.Module): + """ + RMSNorm used in Mega implementation. Differs from T5's RMSNorm by applying the weight prior to taking the square + root (as opposed to after in T5) + """ + + def __init__(self, number_features, eps=1e-6, affine=True): + super().__init__() + self.num_features = number_features + self.eps = eps + self.affine = affine + if affine: + self.weight = nn.Parameter(torch.Tensor(self.num_features)) + else: + self.register_parameter("weight", None) + + def forward(self, input): + mean_square = torch.mean(torch.square(input), dim=-1, keepdim=True) + if self.weight is not None: + input = input * self.weight + + input * torch.rsqrt(mean_square + self.eps) + return input + + +class MegaScaleNorm(nn.Module): + """ + Scale normalization introduced in MEGA which is similar to RMSNorm, but uses a single parameter for scalar + multiplication instead of a vector, and applies over a specified dimension + """ + + def __init__(self, dim, eps=1e-6, affine=True): + super().__init__() + self.dim = dim + self.eps = eps + self.affine = affine + if affine: + self.scalar = nn.Parameter(torch.Tensor(1)) + else: + self.register_parameter("scalar", None) + + def forward(self, input): + mean_square = torch.mean(torch.square(input), dim=self.dim, keepdim=True) + if self.scalar is not None: + input = self.scalar * input + + output = input * torch.rsqrt(mean_square + self.eps) + return output + + +class MegaSequenceNorm(nn.Module): + """ + A wrapper class for various layer normalization options used in Mega. Used to handle differences in expectations on + input axis locations for different normalization methods. + """ + + def __init__(self, norm_type, embedding_dim, eps=1e-5, affine=True, export=False): + super().__init__() + if norm_type == "layernorm": + self.norm = nn.LayerNorm(embedding_dim, eps, elementwise_affine=affine) + elif norm_type == "scalenorm": + self.norm = MegaScaleNorm(dim=-1, eps=eps, affine=affine) + elif norm_type == "rmsnorm": + self.norm = MegaRMSNorm(embedding_dim, eps=eps, affine=affine) + elif norm_type == "batchnorm": + self.norm = nn.BatchNorm1d(embedding_dim, eps=eps, affine=affine) + elif norm_type == "syncbatchnorm": + self.norm = nn.SyncBatchNorm(embedding_dim, eps=eps, affine=affine) + else: + raise ValueError("Unknown norm type: {}".format(norm_type)) + + def forward(self, input): + if isinstance(self.norm, nn.modules.batchnorm._BatchNorm): + if input.dim() != 3: + raise ValueError("BatchNorm inputs must be exactly 3-dimensional") + input = input.permute(1, 2, 0) + input = self.norm(input) + return input.permute(2, 0, 1) + else: + return self.norm(input) + + +# add this layernorm class to ALL_LAYERNORM_LAYERS +ALL_LAYERNORM_LAYERS.append(MegaSequenceNorm) + + +class MegaMultiDimensionDampedEma(nn.Module): + """ + Mega's Exponential Moving Average layer, largely left unmodified from the original repo with the exception of + variable names and moving away from the stateful representation of incremental decoding state. See + "https://arxiv.org/abs/2209.10655" for more details. + """ + + def __init__(self, config: MegaConfig): + super().__init__() + + self.config = config + + self.embed_dim = config.hidden_size + self.ndim = config.ema_projection_size + self.bidirectional = config.bidirectional + self.truncation = config.truncation + self.scale = math.sqrt(1.0 / self.ndim) + + kernel_dim = 2 * config.hidden_size if self.bidirectional else config.hidden_size + # renamed delta (damping_factor) and alpha (decay_factor) to be more descriptive of what the parameters are doing + self.damping_factor = nn.Parameter(torch.Tensor(kernel_dim, self.ndim, 1)) + self.decay_factor = nn.Parameter(torch.Tensor(kernel_dim, self.ndim, 1)) + # renamed gamma (kernel_projection_matrix) and beta (ema_expansion_matrix) respectively to avoid HF renaming + # things and align with the paper's description of these params' behavior + self.ema_expansion_matrix = nn.Parameter(torch.Tensor(kernel_dim, self.ndim, 1)) + self.kernel_projection_matrix = nn.Parameter(torch.Tensor(kernel_dim, self.ndim)) + # renamed omega to residual_weight to describe what it's doing + self.residual_weight = nn.Parameter(torch.Tensor(config.hidden_size)) + self._kernel = None + self._coeffs = None + + def _compute_ema_coefficients(self): + self._coeffs = None + # convert the alpha and delta parameters (kernel_dim x EMA projection size x 1) to [0, 1] with sigmoid + damping_factor = torch.sigmoid(self.damping_factor) + decay_factor = torch.sigmoid(self.decay_factor) + previous_timestep_weight = 1.0 - damping_factor * decay_factor + return damping_factor, previous_timestep_weight + + def _compute_efficient_ema_kernel(self, length: int): + # computes the kernel used for efficient damped EMA applied via FFT convolution + self._kernel = None + # p and q have shape (kernel_dim x ema_projection_size x 1) + damping_factor, previous_timestep_weight = self._compute_ema_coefficients() + # extend the kernel to (kernel_dim X ema_projection_size X sequence_length) and + # multiply q by sequential ints up to the sequence length + vander = torch.arange(length).to(damping_factor).view(1, 1, length) * torch.log(previous_timestep_weight) + kernel = (damping_factor * self.ema_expansion_matrix) * torch.exp(vander) + # (kernel_dim X ema_projection_size X sequence_length) -> (kernel_dim, sequence_length) + return torch.einsum("dnl,dn->dl", kernel, self.kernel_projection_matrix * self.scale) + + def get_ema_coefficients(self): + if self.training: + return self._compute_ema_coefficients() + else: + if self._coeffs is None: + self._coeffs = self._compute_ema_coefficients() + return self._coeffs + + def get_ema_kernel(self, length: int): + kernel_size = length if self.truncation is None else min(self.truncation, length) + if self.training: + return self._compute_efficient_ema_kernel(kernel_size) + else: + if self._kernel is None or self._kernel.size(-1) < kernel_size: + self._kernel = self._compute_efficient_ema_kernel(kernel_size) + return self._kernel[..., :kernel_size] + + def fft_convolution(self, inputs, kernel, length): + # this is a wrapper for repeated use of EMA calculation via FFT (fast Fourier transform) convolution + inputs_fft = torch.fft.rfft(inputs.float(), n=2 * length) + kernel_fft = torch.fft.rfft(kernel.float(), n=2 * length) + convolved_sequence = torch.fft.irfft(inputs_fft * kernel_fft, n=2 * length) + return convolved_sequence + + def ema_step(self, inputs, length, past_state=None): + if length == 1: + return self.one_ema_step(inputs, past_state=past_state) + + # (kernel_dim X ema_projection_size X 1) + damping_factor, previous_timestep_weight = self.get_ema_coefficients() + # (kernel_dim X ema_projection_size X 1+sequence_length) + vander = torch.arange(length + 1).to(damping_factor).view(1, 1, length + 1) * torch.log( + previous_timestep_weight + ) + vander = torch.exp(vander) + if past_state is not None: + # (kernel_dim X ema_projection_size X sequence_length) * (kernel_dim X ema_projection_size X 1) + # -> (kernel_dim X ema_projection_size X sequence_length) + past_ema_proj = vander[:, :, 1:] * (self.kernel_projection_matrix * self.scale).unsqueeze(-1) + # past_state will be (batch_size, kernel_dim, ema_projection_size) + past_ema_state = torch.einsum("bdn,dnl->bdl", past_state, past_ema_proj) + # (kernel_dim X ema_projection_size) * (batch_size X kernel_dim X ema_projection_size) + # -> (batch_size X kernel_dim X ema_projection_size) + past_vandermonde = vander[:, :, -1] * past_state + else: + past_ema_state = None + past_vandermonde = None + + # (kernel_dim X ema_projection_size X sequence_length) + vander = vander[:, :, :-1] + kernel = (damping_factor * self.ema_expansion_matrix) * vander + kernel_proj = torch.einsum("dnl,dn->dl", kernel, self.kernel_projection_matrix * self.scale) + + ema_output = self.fft_convolution(inputs, kernel_proj, length=length)[..., 0:length] + ema_output = ema_output.type_as(inputs) + if past_ema_state is not None: + ema_output = ema_output + past_ema_state + + updated_hidden_state = torch.einsum("bdl,dnl->bdn", inputs, torch.flip(kernel, dims=[2])) + if past_vandermonde is not None: + updated_hidden_state = updated_hidden_state + past_vandermonde + # return a tuple: + # (sequence_length, batch_size, kernel_dim) + # (batch_size, kernel_dim, ema_projection_size) + return ema_output.permute(2, 0, 1), updated_hidden_state + + def one_ema_step(self, inputs, past_state=None): + damping_factor, previous_timestep_weight = self.get_ema_coefficients() + # (kernel_dim X ema_projection_size) x (batch_size X kernel_dim X 1) + # -> (batch_size X kernel_dim X ema_projection_size) + updated_state = (damping_factor * self.ema_expansion_matrix).squeeze(-1) * inputs + if past_state is not None: + updated_state = updated_state + previous_timestep_weight.squeeze(-1) * past_state + # (batch_size X kernel_dim) + out = torch.einsum("bdn,dn->bd", updated_state, self.kernel_projection_matrix * self.scale) + # (1 X batch_size X kernel_dim), (batch_size X kernel_dim X ema_projection_size) + return out.unsqueeze(0), updated_state + + def forward( + self, + inputs, + attention_mask: Optional[torch.Tensor] = None, + prev_state: Optional[torch.Tensor] = None, + use_cache: bool = False, + ) -> torch.Tensor: + """ + Mega's exponential moving average (EMA) sub-layer applied prior to single-headed (traditional) self-attention + + Args: + inputs (`torch.Tensor` of shape `(sequence_length, batch_size, hidden_size)`): + Hidden state / embedding input to update via EMA based on FFT convolution + attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indicates which inputs are to be ignored (mostly due to padding), where elements are either 1 for *not + masked* or 0 for *masked* + prev_state (`torch.Tensor` of shape `(batch_size, config.ndim)`, *optional*): + The hidden state returned from the previous timestep during incremental decoding. + use_cache (`bool`, default `False`): + Whether to perfom incremental decoding; uses `prev_state` as the prior timestep, and returns the + updated EMA hidden state for use in the next step + + Returns: + `tuple(torch.FloatTensor)` containing various elements depending on configuration ([`MegaConfig`]) and + inputs: + - **hidden_states** (`torch.FloatTensor` of shape `(sequence_length, batch_size, hidden_size)`) -- Hidden + states updated by EMA, with same shapes as inputs + - **updated_state** (*optional*, returned when `use_cache=True`) `torch.FloatTensor of shape `(batch_size, + config.ndim)` -- The incremental EMA state for use in the next step of incremental decoding + """ + + seq_len, bsz, embed_dim = inputs.size() + if embed_dim != self.embed_dim: + raise ValueError( + f"Unexpected embedding dimension received: input is {embed_dim}, model expects {self.embed_dim}" + ) + + # sequence_length X batch_size X hidden_size + residual = inputs * self.residual_weight + + # (sequence_length x batch_size x hidden_size) -> (batch_size x hidden_size x sequence_length) + inputs = inputs.permute(1, 2, 0) + # mask the input: output is a tensor with 0 in the masked positions + if attention_mask is not None: + inputs = inputs * (attention_mask.unsqueeze(1).type_as(inputs)) + + if self.bidirectional and use_cache: + raise RuntimeError("Bidirectional EMA does not support incremental state") + + if use_cache: + out, updated_state = self.ema_step(inputs, seq_len, past_state=prev_state) + + # (batch_size X hidden_size) -> (1 x batch_size x hidden_size) + out = F.silu(out + residual) + + # if incremental decoding, return the new state along with the output + return out, updated_state + else: + # (hidden_size x sequence_length) + kernel = self.get_ema_kernel(seq_len) + fft_len = seq_len + s_index = 0 + kernel_size = kernel.size(1) + if self.bidirectional: + # split the kernel for each direction of EMA + k1, k2 = torch.split(kernel, [self.embed_dim, self.embed_dim], dim=0) + # (hidden_size X 2*sequence_length - 1) + kernel = F.pad(k1, (kernel_size - 1, 0)) + F.pad(k2.flip(-1), (0, kernel_size - 1)) + inputs = F.pad(inputs, (kernel_size - 1, 0)) + fft_len = fft_len + kernel_size - 1 + s_index = 2 * kernel_size - 2 + + ema_output = self.fft_convolution(inputs, kernel, length=fft_len)[..., s_index : s_index + seq_len] + ema_output = ema_output.type_as(inputs) + # (batch_size X hidden_size X sequence_length) -> (sequence_length X batch_size X hidden_size) + gated_ema_output = F.silu(ema_output.permute(2, 0, 1) + residual) + + return gated_ema_output, None + + +class MegaGatedCrossAttention(nn.Module): + """ + Gated Structured State Attention for use in encoder-decoder model. See Mega paper for more details. Only + modifications from original implementation are variable names, removing the unnecessary `before_attn_fn` and + `static_kv` arguments, and the stateful representation of incremental decoder state. + """ + + def __init__(self, config: MegaConfig): + super().__init__() + + self.config = config + self.activation = ACT2FN[self.config.activation] + self.attention_activation = self.config.attention_activation + self.scaling = ( + self.config.shared_representation_size**-0.5 if self.attention_activation == "softmax" else None + ) + + self.dropout = MegaDropout(self.config.dropout_prob, is_featurewise=self.config.use_feature_dropout) + self.hidden_dropout = MegaDropout( + self.config.hidden_dropout_prob, is_featurewise=self.config.use_feature_dropout + ) + # Attention dropout is standard dropout + self.attention_dropout = MegaDropout(self.config.attention_probs_dropout_prob, is_featurewise=False) + + self.prenorm = self.config.normalize_before_mega + self.norm = MegaSequenceNorm( + self.config.normalization_type, self.config.hidden_size, affine=self.config.norm_affine + ) + + self.k_proj = nn.Linear(self.config.hidden_size, self.config.shared_representation_size) + self.v_proj = nn.Linear(self.config.hidden_size, self.config.hidden_size) + self.q_proj = nn.Linear( + self.config.hidden_size, 2 * self.config.hidden_size + self.config.shared_representation_size + ) + self.h_proj = nn.Linear(self.config.hidden_size, self.config.hidden_size) + + if self.config.relative_positional_bias == "simple": + self.rel_pos_bias = MegaSimpleRelativePositionalBias(config) + elif self.config.relative_positional_bias == "rotary": + self.rel_pos_bias = MegaRotaryRelativePositionalBias(config) + else: + raise ValueError("unknown relative position bias: {}".format(self.config.relative_positional_bias)) + + self.softmax = nn.Softmax(dim=-1) + + def element_attention(self, query, key, key_padding_mask, pidx): + bsz, src_len, _ = key.size() + tgt_len = query.size(1) if pidx is None else pidx + 1 + if key_padding_mask is not None: + # (batch_size X source_sequence_length) --> (batch_size X 1 X 1) + lengths = key_padding_mask.sum(dim=-1).view(bsz, 1, 1) + else: + lengths = src_len + + # (target_sequence_length X source_sequence_length) + bias = self.rel_pos_bias(max(tgt_len, src_len))[:, :src_len] + if pidx is not None: + if query.size(1) != 1: + raise ValueError("Position offset provided with queries longer than 1 token") + # source_sequence_length + bias = bias[pidx] + else: + # (target_sequence_length X source_sequence_length) + bias = bias[:tgt_len] + + # (batch_size X target_sequence_length X source_sequence_length) + qk = torch.bmm(query, key.transpose(1, 2)) / lengths + bias + + attn_weights = ACT2FN[self.attention_activation](qk).type_as(qk) + + if key_padding_mask is not None: + attn_weights = attn_weights * key_padding_mask.unsqueeze(1) + + return attn_weights + + def softmax_attention(self, query, key, key_padding_mask, pidx): + bsz, src_len, _ = key.size() + tgt_len = query.size(1) if pidx is None else pidx + 1 + + # (target_sequence_length X source_sequence_length) + bias = self.rel_pos_bias(max(tgt_len, src_len))[:, :src_len] + if pidx is not None: + if query.size(1) != 1: + raise ValueError("Position offset provided with queries longer than 1 token") + # source_sequence_length + bias = bias[pidx] + else: + # (target_sequence_length X source_sequence_length) + bias = bias[:tgt_len] + + # scaled attention + query = query * self.scaling + # (batch_size X target_sequence_length X source_sequence_length) + qk = torch.bmm(query, key.transpose(1, 2)) + bias + + if key_padding_mask is not None: + qk = qk.masked_fill((1 - key_padding_mask).unsqueeze(1).to(torch.bool), float("-inf")) + + attn_weights = self.softmax(qk).type_as(qk) + return attn_weights + + def forward( + self, + query, + key: Optional[torch.Tensor], + value: Optional[torch.Tensor], + key_padding_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """ + Gated cross-attention used in Mega + + Args: + query (`torch.Tensor` of shape `(target_sequence_length, batch_size, hidden_size)`): + The self (or target) sequence input used as query inputs for cross-attention + key (`torch.Tensor` of shape `(source_sequence_length, batch_size, hidden_size)`): + The cross (or source) sequence input with shape used as keys in cross-attention + value (`torch.Tensor` of shape `(source_sequence_length, batch_size, hidden_size)`): + The cross (or source) sequence input with shape used as values in cross-attention + key_padding_mask (`torch.LongTensor` of shape `(batch_size, source_sequence_length)`, *optional*): + Padding mask corresponding to the source sequence, where entries are 1 for *not masked* and 0 for + *masked* tokens + past_key_values (`tuple(torch.FloatTensor)`, *optional*): + If provided, the hidden state returned from the previous timestep during incremental decoding; expects + that prior cross-attention keys and values will be the last two items in the tuple + output_attentions (`bool`, defaults to `False`): + Whether or not to return the cross-attention weights. + use_cache (`bool`, defaults to `False`): + Whether to perfom incremental decoding; uses `prev_state` as the prior timestep, and returns the + updated EMA hidden state for use in the next step + + Returns: + `tuple(torch.FloatTensor)` containing various elements depending on configuration ([`MegaConfig`]) and + inputs: + - **hidden_states** (`torch.FloatTensor` of shape `(target_sequence_length, batch_size, hidden_size)`) -- + Hidden states from target sequence updated by gated cross-attention + - **attn_weights** (*optional*, returned when `output_attentions=True`) `torch.FloatTensor` of shape + `(batch_size, source_sequence_length, target_sequence_length)` -- The pairwise cross-attention weights + corresponding to each token in the source and target sequences + - **cross_key** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape `(batch_size, + source_sequence_length, config.shared_representation_size)` -- The cross-attention key state for use in + the next step of incremental decoding + - **cross_value** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape `(batch_size, + source_sequence_length, config.hidden_size)` -- The cross-attention value state for use in the next step + of incremental decoding + """ + + seq_len, bsz, embed_dim = query.size() + if embed_dim != self.config.hidden_size: + raise ValueError( + f"Unexpected embedding dimension received: input is {embed_dim} but expected {self.config.hidden_size}" + ) + + if past_key_values is not None: + # make sure the inputs only have a sequence length of 1 if we're doing incremental decoding + if seq_len != 1: + raise ValueError(f"Incremental decoding requested with self-sequence length > 1: {seq_len}") + # expect past_key_values to have (self_key, self_value, self_ema, cross_key, cross_value) + prev_cross_key, prev_cross_value = past_key_values[-2:] + key = value = None + + # use the self-attention cache to get the position id of the current step + prev_self_key = past_key_values[0] + num_incremental_steps = prev_self_key.size(1) + 1 + else: + prev_cross_key = prev_cross_value = None + # we still need the position id if we're doing incremental decoding (past_key_values will be None for the first step) + num_incremental_steps = 0 if use_cache and (seq_len == 1) else None + + full_query = query + if self.prenorm: + full_query = self.norm(full_query) + + # (target_sequence_length X batch_size X 2*hidden_size + shared_representation_size) + query_projected = self.q_proj(full_query) + # split the query projections into separate components + # - residual_weight is passed through sigmoid and sent through elementwise multiplication to the gated/weighted targets prior to being added to the query directly + # - target_gate is a silu-gated tensor that is multiplied by the attention-weighted target below prior to residual connection + # - attention_query is the part that is passed to the attention function + residual_weight, target_gate, attention_query = torch.split( + query_projected, + [self.config.hidden_size, self.config.hidden_size, self.config.shared_representation_size], + dim=-1, + ) + + # (target_sequence_length X batch_size X hidden_size) + residual_weight = torch.sigmoid(residual_weight) + target_gate = F.silu(target_gate) + + if key is None: + if value is not None: + raise ValueError("Key and value must be `None` simultaneously") + projected_key = projected_value = None + else: + # (source_sequence_length X batch_size X shared_representation_size) + projected_key = self.k_proj(key) + # (source_sequence_length X batch_size X hidden_size) + projected_value = self.activation(self.v_proj(key)) + + # (target_sequence_length X batch_size X shared_representation_size) + # -> (batch_size X target_sequence_length X shared_representation_size) + attention_query = attention_query.transpose(0, 1) + if projected_key is not None: + projected_key = projected_key.transpose(0, 1) + if projected_value is not None: + projected_value = projected_value.transpose(0, 1) + + # if we're doing incremental decoding, k and v are None and need to be overwritten with past values + if past_key_values is not None: + projected_key = prev_cross_key + projected_value = prev_cross_value + + # if we're returning the cache for later use, store these now for later return (can be done without having past_key_values provided) + if use_cache: + updated_cross_key = projected_key + updated_cross_value = projected_value + + ctx_len = projected_key.size(1) + # This is part of a workaround to get around fork/join parallelism + # not supporting Optional types. + if key_padding_mask is not None and key_padding_mask.dim() == 0: + key_padding_mask = None + + if key_padding_mask is not None: + if key_padding_mask.size(0) != bsz: + raise ValueError("Key padding mask does not align on the batch dimension") + if key_padding_mask.size(1) != ctx_len: + raise ValueError("Key padding mask does not align on the sequence length dimension") + + if self.attention_activation == "softmax": + attn_weights = self.softmax_attention( + attention_query, projected_key, key_padding_mask, num_incremental_steps + ) + else: + attn_weights = self.element_attention( + attention_query, projected_key, key_padding_mask, num_incremental_steps + ) + + projected_value = self.hidden_dropout(projected_value, batch_first=True) + kernel = self.attention_dropout(attn_weights) + # (batch_size X target_sequence_length X hidden_size) + # -> (target_sequence_length X batch_size X hidden_size) + weighted_targets = torch.bmm(kernel, projected_value).transpose(0, 1) + # (target_sequence_length X batch_size X hidden_size) + weighted_targets = self.activation(self.h_proj(weighted_targets * target_gate)) + weighted_targets = self.dropout(weighted_targets) + out = torch.addcmul(query, residual_weight, weighted_targets - query) + + if not self.prenorm: + out = self.norm(out) + + outputs = (out, attn_weights) if output_attentions else (out,) + if use_cache: + outputs = outputs + (updated_cross_key, updated_cross_value) + + return outputs + + +class MegaMovingAverageGatedAttention(nn.Module): + """ + Pure PyTorch implementation of Mega block; see https://arxiv.org/abs/2209.10655 and original fairseq implementation + at https://github.com/facebookresearch/mega (copyright Meta Research, licensed under MIT License) + + Differences from original implementation include hidden state refactor and fixed inconsistency with additive / + multiplicative attention masks + """ + + def __init__(self, config: MegaConfig): + super().__init__() + self.config = config + self.activation = ACT2FN[self.config.activation] + self.scaling = ( + self.config.shared_representation_size**-0.5 if self.config.attention_activation == "softmax" else None + ) + self.dropout = MegaDropout(self.config.dropout_prob, is_featurewise=self.config.use_feature_dropout) + self.hidden_dropout = MegaDropout( + self.config.hidden_dropout_prob, is_featurewise=self.config.use_feature_dropout + ) + # attention dropout is standard dropout + self.attention_dropout = MegaDropout(self.config.attention_probs_dropout_prob, is_featurewise=False) + + self.norm = MegaSequenceNorm( + self.config.normalization_type, self.config.hidden_size, affine=self.config.norm_affine + ) + self.ema_gate = MegaMultiDimensionDampedEma(config) + + self.v_proj = nn.Linear(self.config.hidden_size, self.config.intermediate_size) + self.mx_proj = nn.Linear( + self.config.hidden_size, + self.config.shared_representation_size + self.config.intermediate_size + 2 * self.config.hidden_size, + ) + self.h_proj = nn.Linear(self.config.intermediate_size, self.config.hidden_size) + + self.qk_weight = nn.Parameter(torch.Tensor(2, self.config.shared_representation_size)) + self.qk_bias = nn.Parameter(torch.Tensor(2, self.config.shared_representation_size)) + + if self.config.relative_positional_bias == "simple": + self.rel_pos_bias = MegaSimpleRelativePositionalBias(config) + elif self.config.relative_positional_bias == "rotary": + self.rel_pos_bias = MegaRotaryRelativePositionalBias(config) + else: + raise ValueError(f"Unknown relative positional bias: {self.config.relative_positional_bias}") + + self.softmax = nn.Softmax(dim=-1) + self.attention_function = ( + self.softmax_attention if self.config.attention_activation == "softmax" else self.element_attention + ) + + def element_attention(self, query, key, padding_mask, causal_mask): + """ + Apply element-wise attention via relu^2 or laplace. Same as original implementation but with standardized + causal attention mask. Expects the Hugging Face standard attention mask paradigm: 1 for not masked, and 0 for + masked. + """ + seq_len = key.size(2) + if padding_mask is not None: + # (batch_size X number of chunks X 1) + lengths = padding_mask.sum(-1, keepdim=True) + # (batch_size X number of chunks X 1 X 1) + lengths = lengths.clamp(min=1.0).unsqueeze(-1) + else: + lengths = seq_len + + if causal_mask is not None: + lengths = causal_mask.sum(dim=-1, keepdim=True) + + # (sequence_length X sequence_length) + bias = self.rel_pos_bias(seq_len) + if seq_len != query.size(2): + if query.size(2) != 1: + raise ValueError("Size mismatch between Q and K in element attention") + # (1 X sequence_length) + bias = bias[-1:] + + # (batch_size X number of chunks X sequence_length X sequence_length) + qk = torch.matmul(query, key.transpose(2, 3)) / lengths + bias + + attn_weights = ACT2FN[self.config.attention_activation](qk).type_as(qk) + + if padding_mask is not None: + attn_weights = attn_weights * padding_mask.unsqueeze(2) + + if causal_mask is not None: + attn_weights = attn_weights * causal_mask + + return attn_weights + + def softmax_attention(self, query, key, padding_mask, causal_mask): + "Standard softmax self-attention, as in the original Transformer paper" + seq_len = key.size(2) + # (sequence_length X sequence_length) + bias = self.rel_pos_bias(seq_len) + if seq_len != query.size(2): + if query.size(2) != 1: + raise ValueError("Size mismatch between Q and K in softmax attention") + # (1 X sequence_length) + bias = bias[-1:] + + # scaled attention + query = query * self.scaling + + # (batch_size x number of chunks x chunk_size x chunk_size) if chunking + # (batch_size x 1 x sequence_length x sequence_length) otherwise + qk = torch.matmul(query, key.transpose(2, 3)) + bias + + # apply causal mask (presumed to be 1/0 for not masked / masked) + # additive, but convert to 0/-inf (which is not explicitly in the Mega source code) + if causal_mask is not None: + additive_causal_mask = torch.zeros_like(causal_mask, dtype=torch.float) + additive_causal_mask = additive_causal_mask.masked_fill((1 - causal_mask).bool(), float("-inf")) + qk = qk + additive_causal_mask + + if padding_mask is not None: + # 1 for tokens which are *not masked* + # 0 for tokens which are *masked* + # replace masked tokens with -inf to make softmax ignore them + # need to invert the padding mask to match what mega original did + padding_mask = 1 - padding_mask + padding_mask_all = padding_mask.all(dim=-1, keepdim=True) + padding_mask = torch.logical_and(padding_mask, ~padding_mask_all) + qk = qk.masked_fill(padding_mask.unsqueeze(2).to(torch.bool), float("-inf")) + + attn_weights = self.softmax(qk).type_as(qk) + return attn_weights + + def forward( + self, + input, + padding_mask: Optional[torch.Tensor] = None, + causal_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[Tuple[torch.Tensor]] = None, + output_attentions=False, + use_cache=False, + ): + """ + Mega's self-attention block, which combines multi-headed EMA with traditional self-attention + + Args: + input (`torch.Tensor` of shape `(sequence_length, batch_size, hidden_size)`): + Hidden states to be updated by Mega's self-attention + padding_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indicates which inputs are to be ignored due to padding, where elements are either 1 for *not masked* + or 0 for *masked* + causal_mask (`torch.LongTensor` of shape `(sequence_length, sequence_length)`, *optional*): + Indicates which inputs are to be ignored due to causal attention, where elements are either 1 for *not + masked* or 0 for *masked* + past_key_values (`tuple(torch.Tensor)`, *optional*): + The hidden states returned from the previous timestep during incremental decoding; expects that + self-attention key, value, and EMA states are the first 3 entries in the tuple + output_attentions (`bool`, default `False`): + Whether to return self-attention weights + use_cache (`bool`, default `False`): + Whether to perfom incremental decoding; uses `past_key_values` as prior state, and returns the updated + states for use in the next step + + Returns: + `tuple(torch.FloatTensor)` containing various elements depending on configuration ([`MegaConfig`]) and + inputs: + - **hidden_states** (`torch.FloatTensor` of shape `(sequence_length, batch_size, hidden_size)`) -- Hidden + states from target sequence updated by Mega's self-attention + - **attn_weights** (*optional*, returned when `output_attentions=True`) `torch.FloatTensor` of shape + `(batch_size, 1, sequence_length, sequence_length)` -- The self-attention weights corresponding to how + each token in the input sequence attends to every other token + - **self_key** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape `(batch_size, + sequence_length, config.shared_representation_size)` -- The self-attention key state for use in the next + step of incremental decoding + - **self_value** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape `(batch_size, + sequence_length, config.hidden_size)` -- The self-attention value state for use in the next step of + incremental decoding + - **self_ema_state** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape + `(batch_size, config.ndim)` The incremental EMA state for use in the next step of incremental decoding. + """ + + seq_len, bsz, embed_dim = input.size() + if embed_dim != self.config.hidden_size: + raise ValueError(f"Input embedding dimension should be {self.config.hidden_size}; received {embed_dim}") + + # store inputs for residual connection and handle pre-norm if requested + residual = input + if self.config.normalize_before_mega: + input = self.norm(input) + + # (sequence_length X batch_size X hidden_size) -> (sequence_length X batch_size X intermediate_size) + value = self.activation(self.v_proj(input)) + + # unpack the incremental state if provided + # assumed to be (self K, self V, self EMA state, cross K, cross V) + # also assumes that incremental decoding is working one token at a time, so input sequence length must be 1 + if self.config.is_decoder and (past_key_values is not None): + if seq_len > 1: + raise ValueError(f"Incremental decoding only supports self sequence length of 1; received {seq_len}") + # the first 3 items in the saved states will be these regardless of whether cross-attention is present + prev_self_key, prev_self_value, prev_ema_state = past_key_values[0:3] + else: + prev_self_key = prev_self_value = prev_ema_state = None + + # ema output is (sequence_length x batch_size x hidden_size) + # updated_ema_state will be None if use_cache=False; otherwise (batch_size, config.ndim) + ema_out, updated_ema_state = self.ema_gate( + input, attention_mask=padding_mask, prev_state=prev_ema_state, use_cache=use_cache + ) + ema_out = self.dropout(ema_out) + + # (sequence_length X batch_size X hidden_size) + # -> (sequence_length X batch_size X 2*hidden_size + config.shared_representation_size + config.intermediate_size) + # - residual_weight -> sigmoid -> applied to residual connection in torch.addcmul + # - query_key_gates -> split into two components: query_key becomes query and key for attention input, gates becomes gating for self-attention output + # - intermediate_state -> added to weighted attention output, sent through activation, and has inputs subtracted during + # torch.addcmul to create the final layer output + base = self.mx_proj(ema_out) + residual_weight, query_key_gates, intermediate_state = torch.split( + base, + [ + self.config.hidden_size, + self.config.shared_representation_size + self.config.intermediate_size, + self.config.hidden_size, + ], + dim=-1, + ) + + # (sequence_length X batch_size X hidden_size) + residual_weight = torch.sigmoid(residual_weight) + + # (sequence_length X batch_size X shared_representation_size + intermediate_size) + query_key_gates = F.silu(query_key_gates) + + # split into two different tensors: one for Q/K usage and the other for gating self-attention + query_key, attention_gate = torch.split( + query_key_gates, [self.config.shared_representation_size, self.config.intermediate_size], dim=-1 + ) + + # (sequence_length X batch_size X shared_representation_size) + # -> (sequence_length X batch_size X 1 X shared_representation_size) + # -> (sequence_length X batch_size X 2 X shared_representation_size) + query_key = query_key.unsqueeze(2) * self.qk_weight + self.qk_bias + + # (sequence_length X batch_size X 2 X shared_representation_size) + # -> 2 tensors of (sequence_length X batch_size X shared_representation_size) + query, key = torch.unbind(query_key, dim=2) + + # (sequence_length X batch_size X dimension) + # -> (batch_size X sequence_length X dimension) + # where `dimension` is either shared_representation_size (queries and keys) or intermediate_size (values) + query = query.transpose(0, 1) + key = key.transpose(0, 1) + value = value.transpose(0, 1) + + if self.config.is_decoder: + # combine history and current to save updated state (if history is provided) + # when chunking is applied, the past states will be None at the end of the chunk, in + # which case, proceed as if no K/V history had been provided + # saved states are stored with shape (batch_size X sequence_length X dimension) + if prev_self_key is not None: + key = torch.cat([prev_self_key, key], dim=1) + if prev_self_value is not None: + value = torch.cat([prev_self_value, value], dim=1) + + # if not chunking, store as-is + if not self.config.use_chunking: + updated_self_key = key + updated_self_value = value + else: + curr_len = key.size(1) % self.config.chunk_size + if curr_len == 0: + # if we're chunking and have reached the end of a chunk, wipe out the saved state + updated_self_key = None + updated_self_value = None + else: + updated_self_key = key + updated_self_value = value + + ctx_len = key.size(1) # potentially differs from seq_len because of incremental decoding + if not self.config.use_chunking: + # if we're not chunking, treat the entire sequence as one long chunk + # (batch_size X sequence_length X dimension) -> (batch_size X 1 X sequence_length X dimension) + query = query.unsqueeze(1) + key = key.unsqueeze(1) + value = value.unsqueeze(1) + if padding_mask is not None: + # (batch_size X sequence_length) -> (batch_size X 1 X sequence_length) + padding_mask = padding_mask.unsqueeze(1) + else: + # otherwise, split the sequences in the batch into `n_chunks` chunks of size `chunk_size` + if seq_len < self.config.chunk_size: + query = query.unsqueeze(1) + else: + # (batch_size X sequence_length X dimension) -> (batch_size X n_chunks X chunk_size X dimension) + n_chunks = seq_len // self.config.chunk_size + query = query.reshape(bsz, n_chunks, self.config.chunk_size, self.config.shared_representation_size) + + if ctx_len < self.config.chunk_size: + key = key.unsqueeze(1) + value = value.unsqueeze(1) + if padding_mask is not None: + padding_mask = padding_mask.unsqueeze(1) + else: + # (batch_size X sequence_length X dimension) -> (batch_size X n_chunks X chunk_size X dimension) + n_chunks = ctx_len // self.config.chunk_size + key = key.reshape(bsz, n_chunks, self.config.chunk_size, self.config.shared_representation_size) + value = value.reshape(bsz, n_chunks, self.config.chunk_size, self.config.intermediate_size) + if padding_mask is not None: + padding_mask = padding_mask.view(bsz, n_chunks, self.config.chunk_size) + + # this is in the original Mega implementation to work around fork/join parallelism not supporting optional types + if padding_mask is not None and padding_mask.dim() == 0: + padding_mask = None + + attn_weights = self.attention_function(query, key, padding_mask=padding_mask, causal_mask=causal_mask) + + value = self.hidden_dropout(value, batch_first=True) + kernel = self.attention_dropout(attn_weights) + + # (batch_size x n_chunks x chunk_size x intermediate_size) -> (sequence_length X batch_size X intermediate_size) + weighted_self_output = ( + torch.matmul(kernel, value).view(bsz, seq_len, self.config.intermediate_size).transpose(0, 1) + ) + + # (sequence_length X batch_size X intermediate_size) -> (sequence_length X batch_size X hidden_size) + weighted_self_output = self.activation(intermediate_state + self.h_proj(weighted_self_output * attention_gate)) + weighted_self_output = self.dropout(weighted_self_output) + # (sequence_length X batch_size X hidden_size) + out = torch.addcmul(residual, residual_weight, weighted_self_output - residual) + + if not self.config.normalize_before_mega: + out = self.norm(out) + + return_values = (out, attn_weights) if output_attentions else (out,) + + if self.config.is_decoder: + return_values = return_values + (updated_self_key, updated_self_value, updated_ema_state) + + return return_values + + +class MegaNormalizedFeedForwardNetwork(nn.Module): + """ + Normalized feed-forward network used in Mega blocks. Left as-is from original Mega repo aside from retrieving args + from Hugging Face config + """ + + def __init__(self, config: MegaConfig): + super().__init__() + + self.config = config + self.hidden_dim = config.nffn_hidden_size + self.act_fn = config.activation + self.activation = ACT2FN[config.activation] + + self.dropout = MegaDropout(self.config.dropout_prob, is_featurewise=self.config.use_feature_dropout) + self.hidden_dropout = MegaDropout( + self.config.nffn_activation_dropout_prob, is_featurewise=self.config.use_feature_dropout + ) + + self.prenorm = self.config.normalize_before_ffn + self.norm = MegaSequenceNorm( + self.config.normalization_type, self.config.hidden_size, affine=self.config.norm_affine + ) + + self.fc1 = nn.Linear(self.config.hidden_size, self.config.nffn_hidden_size) + self.fc2 = nn.Linear(self.config.nffn_hidden_size, self.config.hidden_size) + + def forward(self, inputs): + residual = inputs + + if self.prenorm: + inputs = self.norm(inputs) + + hidden = self.activation(self.fc1(inputs)) + hidden = self.hidden_dropout(hidden) + output = self.fc2(hidden) + output = self.dropout(output) + output = output + residual + + if not self.prenorm: + output = self.norm(output) + + return output + + +class MegaBlock(nn.Module): + def __init__(self, config: MegaConfig): + super().__init__() + self.seq_len_dim = 1 + self.mega_layer = MegaMovingAverageGatedAttention(config) + self.nffn = MegaNormalizedFeedForwardNetwork(config) if config.use_normalized_ffn else None + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + if not self.is_decoder: + raise ValueError(f"{self} should be used as a decoder model if cross attention is added") + self.cross_attn = MegaGatedCrossAttention(config) + else: + self.cross_attn = None + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + causal_mask: Optional[torch.LongTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[torch.FloatTensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor]: + """ + A single Mega layer: either encoder or decoder, with optional cross-attention and optional normalized + feed-forward layer + + Args: + hidden_states (`torch.Tensor` of shape `(target_sequence_length, batch_size, hidden_size)`): + Hidden states to be updated by the Mega block + attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indicates which entries in the self/target sequence are to be ignored (mostly due to padding), where + elements are either 1 for *not masked* or 0 for *masked*. Causal attention is enforced internally. + causal_mask (`torch.LongTensor` of shape `(sequence_length, sequence_length)`, *optional*): + Indicates which inputs are to be ignored due to causal attention, where elements are either 1 for *not + masked* or 0 for *masked* + encoder_hidden_states (`torch.Tensor`, of shape `(source_sequence_length, batch_size, hidden_size)`, *optional*): + Encoder hidden states to be used for cross-attention (and required for encoder-decoder model setup) + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, source_sequence_length)`, *optional*): + Indicates which entries in the cross/source sequence are to be ignored (mostly due to padding), where + elements are either 1 for *not masked* or 0 for *masked*. + past_key_value (`tuple(torch.Tensor)`, *optional*): + The hidden states returned from the previous timestep during incremental decoding; expects that + self-attention key, value, and EMA states are the first 3 entries in the tuple, and (if doing + cross-attention) cross-attention key and value are the last 2 entries in the tuple + output_attentions (`bool`, default `False`): + Whether to return self-attention weights + use_cache (`bool`, default `False`): + Whether to perfom incremental decoding; uses `past_key_value` as prior state, and returns the updated + states for use in the next step + + Returns: + `tuple(torch.FloatTensor)` containing various elements depending on configuration ([`MegaConfig`]) and + inputs: + - **hidden_states** (`torch.FloatTensor` of shape `(target_sequence_length, batch_size, hidden_size)`) -- + Hidden states from target sequence updated by Mega + - **self_attn_weights** (*optional*, returned when `output_attentions=True`) `torch.FloatTensor` of shape + `(batch_size, 1, target_sequence_length, target_sequence_length)` -- The self-attention weights + corresponding to how each token in the input sequence attends to every other token + - **cross_attn_weights** (*optional*, returned when `output_attentions=True` and + `config.add_cross_attention=True`) `torch.FloatTensor` of shape `(batch_size, source_sequence_length, + target_sequence_length)` -- Pairwise cross-attention weights between every entry in the source sequence + and target sequence + - **self_key** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape `(batch_size, + sequence_length, config.shared_representation_size)` -- The self-attention key state for use in the next + step of incremental decoding + - **self_value** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape `(batch_size, + sequence_length, config.hidden_size)` -- The self-attention value state for use in the next step of + incremental decoding + - **self_ema_state** (*optional*, returned when `use_cache=True`) `torch.FloatTensor` of shape + `(batch_size, config.ndim)` The incremental EMA state for use in the next step of incremental decoding. + - **cross_key** (*optional*, returned when `use_cache=True` and `config.is_decoder=True`) + `torch.FloatTensor` of shape `(batch_size, source_sequence_length, config.shared_representation_size)` -- + The cross-attention key state for use in the next step of incremental decoding + - **cross_value** (*optional*, returned when `use_cache=True` and `config.is_decoder=True`) + `torch.FloatTensor` of shape `(batch_size, source_sequence_length, config.hidden_size)` -- The + cross-attention value state for use in the next step of incremental decoding + """ + + # incremental decoding in the MegaMultiDimensionDampedEma module requires that the attention mask has the same + # sequence length as the input tensor; if we're caching incremental states, we assume the input + # sequence length is 1 (Mega will break otherwise), so we take the padding mask for the final + # token in the input (mask is received as [batch X sequence length]) + if use_cache and (past_key_value is not None) and (attention_mask is not None): + mega_padding_mask = attention_mask[:, -1].unsqueeze(-1) + else: + mega_padding_mask = attention_mask + + mega_outputs = self.mega_layer( + input=hidden_states, + padding_mask=mega_padding_mask, + causal_mask=causal_mask, + past_key_values=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + new_hidden_states = mega_outputs[0] + self_key, self_value, self_ema_state = mega_outputs[-3:] if use_cache else (None, None, None) + self_attention_weights = mega_outputs[1] if output_attentions else None + + # optional cross attention + if self.cross_attn is not None: + if encoder_hidden_states is None: + raise ValueError("Requested cross-attention without providing encoder hidden states") + + cross_attn_outputs = self.cross_attn( + query=new_hidden_states, + key=encoder_hidden_states, + value=encoder_hidden_states, + key_padding_mask=encoder_attention_mask, + past_key_values=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + # update the hidden state from cross attention + new_hidden_states = cross_attn_outputs[0] + # store cross-attention k/v if caching + cross_key, cross_value = cross_attn_outputs[-2:] if use_cache else (None, None) + cross_attention_weights = cross_attn_outputs[1] if output_attentions else None + + # optional NFFN follows cross attention + if self.nffn is not None: + new_hidden_states = self.nffn(new_hidden_states) + + outs = (new_hidden_states,) + if output_attentions: + outs = outs + (self_attention_weights,) + if self.cross_attn is not None: + outs = outs + (cross_attention_weights,) + + if use_cache: + new_key_values = ( + self_key, + self_value, + self_ema_state, + ) + if self.cross_attn is not None: + new_key_values = new_key_values + (cross_key, cross_value) + + outs = outs + (new_key_values,) + + return outs + + +# copied from transformers.models.roberta.modeling_roberta.RobertaPooler with Roberta->Mega +class MegaPooler(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class MegaPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = MegaConfig + base_model_prefix = "mega" + supports_gradient_checkpointing = False + _no_split_modules = [] + + def _init_weights(self, module): + """Initialize the weights""" + if isinstance(module, MegaMultiDimensionDampedEma): + with torch.no_grad(): + # delta & alpha + nn.init.normal_(module.damping_factor, mean=0.0, std=self.config.ema_delta_alpha_range) + nn.init.normal_(module.decay_factor, mean=0.0, std=self.config.ema_delta_alpha_range) + # beta [1, -1, 1, -1, ...] seems more stable. + val = torch.ones(self.config.ema_projection_size, 1) + if self.config.ema_projection_size > 1: + idx = torch.tensor(list(range(1, self.config.ema_projection_size, 2))) + val.index_fill_(0, idx, -1.0) + module.ema_expansion_matrix.normal_(mean=0.0, std=self.config.ema_beta_range).add_(val) + # gamma & omega + nn.init.normal_(module.kernel_projection_matrix, mean=0.0, std=self.config.ema_gamma_omega_range) + nn.init.normal_(module.residual_weight, mean=0.0, std=self.config.ema_gamma_omega_range) + elif isinstance(module, MegaSimpleRelativePositionalBias): + nn.init.normal_(module.rel_pos_bias, mean=0.0, std=self.config.initializer_range) + elif isinstance(module, MegaRotaryRelativePositionalBias): + nn.init.normal_(module.alpha, mean=0.0, std=self.config.initializer_range) + nn.init.normal_(module.b_param, mean=0.0, std=self.config.initializer_range) + elif isinstance(module, MegaScaleNorm): + if self.config.norm_affine: + nn.init.constant_(module.scalar, 1.0) + elif isinstance(module, MegaRMSNorm): + if self.config.norm_affine: + nn.init.constant_(module.weight, 1.0) + elif isinstance(module, MegaMovingAverageGatedAttention): + # linear layers covered separately by the generic nn.Linear init below + nn.init.normal_(module.qk_weight, mean=0.0, std=self.config.initializer_range) + nn.init.constant_(module.qk_bias, 0.0) + elif isinstance(module, nn.Linear): + # initializes all linear layers in the entire network + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + def update_keys_to_ignore(self, config, del_keys_to_ignore): + """Remove some keys from ignore list""" + if not config.tie_word_embeddings: + # must make a new list, or the class variable gets modified! + self._keys_to_ignore_on_save = [k for k in self._keys_to_ignore_on_save if k not in del_keys_to_ignore] + self._keys_to_ignore_on_load_missing = [ + k for k in self._keys_to_ignore_on_load_missing if k not in del_keys_to_ignore + ] + + +MEGA_START_DOCSTRING = r""" + + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`MegaConfig`]): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +MEGA_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `({0})`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): + Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: + + - 0 corresponds to a *sentence A* token, + - 1 corresponds to a *sentence B* token. + This parameter can only be used when the model is initialized with `add_token_type_embeddings` parameter + set to `True`. All the value in this tensor should be always < config.type_vocab_size. + + [What are token type IDs?](../glossary#token-type-ids) + inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare MEGA Model transformer outputting raw hidden-states without any specific head on top.", + MEGA_START_DOCSTRING, +) +class MegaModel(MegaPreTrainedModel): + """ + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added after self-attention, following the architecture described in *Mega: Moving Average + Equipped Gated Attention*_ by Xuezhe Ma, Chunting Zhou, Xiang Kong, Junxian He, Liangke Gui, Graham Neubig, + Jonathan May, and Luke Zettlemoyer + + To behave as a decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to + `True` and `bidirectional` set to `False`. To be used in a Seq2Seq model, the model needs to initialized with both + `is_decoder=True` and `bidirectional=False` argument as well as `add_cross_attention` set to `True`; an + `encoder_hidden_states` is then expected as an input to the forward pass. + + .. _*Mega: Moving Average Equipped Gated Attention*: https://arxiv.org/abs/2209.10655 + + """ + + _keys_to_ignore_on_load_missing = [] + + def __init__(self, config: MegaConfig, add_pooling_layer=True): + super().__init__(config) + self.config = config + + self.embedding_layer = MegaEmbeddings(config) + self.layers = nn.ModuleList([MegaBlock(config) for _ in range(config.num_hidden_layers)]) + + self.pooler = MegaPooler(config) if add_pooling_layer else None + + # Initialize weights and apply final processing (retained from RoBERTa code) + self.post_init() + + def get_input_embeddings(self): + return self.embedding_layer.word_embeddings + + def set_input_embeddings(self, value): + self.embedding_layer.word_embeddings = value + + @add_start_docstrings_to_model_forward(MEGA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=BaseModelOutputWithPoolingAndCrossAttentions, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + token_type_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPoolingAndCrossAttentions]: + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + """ + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.use_chunking and (input_ids.size(1) > self.config.chunk_size): + if input_ids.size(1) % self.config.chunk_size != 0: + raise ValueError( + f"config.use_chunking is activated; input sequence length must be shorter than or a multiple of config.chunk_size\nreceived sequence length of {input_ids.size(1)} with chunk size {self.config.chunk_size}" + ) + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + device = input_ids.device + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + device = inputs_embeds.device + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + batch_size, sequence_length = input_shape + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + + # Mega expects the causal mask to be a 2D square matrix of (from) x (to) over the input sequence length + # the HF utility function generates a 3D causal mask which includes batch size, so we'll create a dummy + # mask with the correct device and all ones + temp_mask_for_extension = torch.ones((1, sequence_length), dtype=torch.long, device=device) + causal_mask = self.create_extended_attention_mask_for_decoder(input_shape, temp_mask_for_extension) + + # get rid of batch dimension in the generated mask; result is (sequence_length X sequence_length) + causal_mask = causal_mask.squeeze(0) + else: + use_cache = False + causal_mask = None + + # if using cache, make sure we have a tuple of tuples which matches the length of our hidden layers + if (past_key_values is not None) and (len(past_key_values) != self.config.num_hidden_layers): + raise ValueError( + f"Received past key/value cache with size mismatch; expected {self.config.num_hidden_layers}, received {len(past_key_values)}" + ) + + # get embeddings (batch X sequence length X embed dim) + embedding_output = self.embedding_layer( + input_ids=input_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds + ) + + # transpose for Mega --> (seq len X batch X embed dim) + hidden_states = embedding_output.transpose(0, 1) + + # we expect encoder hidden states to also have batch first in line + # with typical Hugging Face behavior (which is also how we return them) + # Mega expects sequence length first, so do the same transpose here + if encoder_hidden_states is not None: + encoder_hidden_states = encoder_hidden_states.transpose(0, 1) + + # pass through mega layers + all_hidden_states = (embedding_output,) if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None + next_decoder_cache = () if use_cache else None + for i, mega_layer in enumerate(self.layers): + current_decoder_cache = past_key_values[i] if past_key_values is not None else None + mega_outputs = mega_layer( + hidden_states=hidden_states, + attention_mask=attention_mask, + causal_mask=causal_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_value=current_decoder_cache, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = mega_outputs[0] + if output_hidden_states: + # store layer-wise hidden states in the way that the user expects + # (seq len X batch X embed dim) --> (batch X seq len X embed dim) + all_hidden_states += (hidden_states.transpose(0, 1),) + if output_attentions: + self_attn_weights = mega_outputs[1] + all_self_attentions += (self_attn_weights,) + if self.config.add_cross_attention: + cross_attn_weights = mega_outputs[2] + all_cross_attentions += (cross_attn_weights,) + if use_cache: + updated_cache = mega_outputs[-1] + next_decoder_cache += (updated_cache,) + + # transpose final hidden states + hidden_states = hidden_states.transpose(0, 1) + + # optional pooling layer + pooled_output = self.pooler(hidden_states) if self.pooler is not None else None + + if not return_dict: + return (hidden_states, pooled_output) + ( + all_hidden_states, + next_decoder_cache, + all_self_attentions, + all_cross_attentions, + ) + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=hidden_states, + pooler_output=pooled_output, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +@add_start_docstrings( + """MEGA Model with a `language modeling` head on top for CLM fine-tuning.""", MEGA_START_DOCSTRING +) +class MegaForCausalLM(MegaPreTrainedModel): + _keys_to_ignore_on_save = [r"lm_head.weight", r"lm_head.bias"] + _keys_to_ignore_on_load_missing = [r"lm_head.weight", r"lm_head.bias"] + _keys_to_ignore_on_load_unexpected = [r"pooler"] + + def __init__(self, config: MegaConfig): + super().__init__(config) + + if not config.is_decoder: + logger.warning("If you want to use `MegaForCausalLM` as a standalone, add `is_decoder=True.`") + + self.mega = MegaModel(config, add_pooling_layer=False) + + if config.add_lm_hidden_dense_layer: + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.hidden_activation = nn.Tanh() + else: + self.dense = None + self.hidden_activation = None + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size) + + # The LM head weights require special treatment only when they are tied with the word embeddings + self.update_keys_to_ignore(config, ["lm_head.weight"]) + + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + @add_start_docstrings_to_model_forward(MEGA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + past_key_values: Tuple[Tuple[torch.FloatTensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in + `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are + ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, MegaForCausalLM, AutoConfig + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("mnaylor/mega-base-wikitext") + >>> config = AutoConfig.from_pretrained("mnaylor/mega-base-wikitext") + >>> config.is_decoder = True + >>> config.bidirectional = False + >>> model = MegaForCausalLM.from_pretrained("mnaylor/mega-base-wikitext", config=config) + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + + >>> prediction_logits = outputs.logits + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if labels is not None: + use_cache = False + + outputs = self.mega( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + if self.dense is not None: + sequence_output = self.dense(sequence_output) + sequence_output = self.hidden_activation(sequence_output) + + prediction_scores = self.lm_head(sequence_output) + + lm_loss = None + if labels is not None: + # we are doing next-token prediction; shift prediction scores and input ids by one + shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss() + lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + + if not return_dict: + output = (prediction_scores,) + outputs[2:] + return ((lm_loss,) + output) if lm_loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=lm_loss, + logits=prediction_scores, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attention_mask=None, **model_kwargs): + input_shape = input_ids.shape + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # cut decoder_input_ids if past is used + if past_key_values is not None: + input_ids = input_ids[:, -1:] + + return {"input_ids": input_ids, "attention_mask": attention_mask, "past_key_values": past_key_values} + + def _reorder_cache(self, past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past + + +@add_start_docstrings("""MEGA Model with a `language modeling` head on top.""", MEGA_START_DOCSTRING) +class MegaForMaskedLM(MegaPreTrainedModel): + _keys_to_ignore_on_save = [r"mlm_head.weight", r"mlm_head.bias"] + _keys_to_ignore_on_load_missing = [r"mlm_head.weight", r"mlm_head.bias"] + _keys_to_ignore_on_load_unexpected = [r"pooler"] + + def __init__(self, config: MegaConfig): + super().__init__(config) + + if config.is_decoder: + logger.warning( + "If you want to use `MegaForMaskedLM`, set `config.is_decoder=False` for " + "bi-directional self-attention." + ) + + self.mega = MegaModel(config, add_pooling_layer=False) + if config.add_lm_hidden_dense_layer: + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.hidden_activation = nn.Tanh() + else: + self.dense = None + self.hidden_activation = None + self.mlm_head = nn.Linear(config.hidden_size, config.vocab_size) + self.dropout = nn.Dropout(config.dropout_prob) + + # The LM head weights require special treatment only when they are tied with the word embeddings + self.update_keys_to_ignore(config, ["mlm_head.weight"]) + + # Initialize weights and apply final processing + self.post_init() + + def get_output_embeddings(self): + return self.mlm_head + + def set_output_embeddings(self, new_embeddings): + self.mlm_head = new_embeddings + + @add_start_docstrings_to_model_forward(MEGA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=MaskedLMOutput, + config_class=_CONFIG_FOR_DOC, + mask="", + expected_output="' Paris'", + expected_loss=0.1, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., + config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + kwargs (`Dict[str, any]`, optional, defaults to *{}*): + Used to hide legacy arguments that have been deprecated. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.mega( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = outputs[0] + if self.dense is not None: + sequence_output = self.dense(sequence_output) + sequence_output = self.hidden_activation(sequence_output) + prediction_scores = self.mlm_head(sequence_output) + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + + if not return_dict: + output = (prediction_scores,) + outputs[2:] + return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output + + return MaskedLMOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + MEGA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled + output) e.g. for GLUE tasks. + """, + MEGA_START_DOCSTRING, +) +class MegaForSequenceClassification(MegaPreTrainedModel): + _keys_to_ignore_on_load_missing = [] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.config = config + + self.mega = MegaModel(config, add_pooling_layer=False) + self.classifier = MegaClassificationHead(config) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(MEGA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=SequenceClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.mega( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = outputs[0] + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.num_labels == 1: + loss = loss_fct(logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + MEGA Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a + softmax) e.g. for RocStories/SWAG tasks. + """, + MEGA_START_DOCSTRING, +) +class MegaForMultipleChoice(MegaPreTrainedModel): + _keys_to_ignore_on_load_missing = [] + + def __init__(self, config): + super().__init__(config) + + self.mega = MegaModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(MEGA_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length")) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=MultipleChoiceModelOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., + num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See + `input_ids` above) + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1] + + flat_input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None + flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + flat_inputs_embeds = ( + inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1)) + if inputs_embeds is not None + else None + ) + + outputs = self.mega( + flat_input_ids, + token_type_ids=flat_token_type_ids, + attention_mask=flat_attention_mask, + inputs_embeds=flat_inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + + if not return_dict: + output = (reshaped_logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return MultipleChoiceModelOutput( + loss=loss, + logits=reshaped_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + MEGA Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for + Named-Entity-Recognition (NER) tasks. + """, + MEGA_START_DOCSTRING, +) +class MegaForTokenClassification(MegaPreTrainedModel): + _keys_to_ignore_on_load_unexpected = [r"pooler"] + _keys_to_ignore_on_load_missing = [] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.mega = MegaModel(config, add_pooling_layer=False) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob + ) + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(MEGA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TokenClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.mega( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +# copied from transformers.models.roberta.modeling_roberta.RobertaClassificationHead with Roberta->Mega +class MegaClassificationHead(nn.Module): + """Head for sentence-level classification tasks.""" + + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + classifier_dropout = ( + config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob + ) + self.dropout = nn.Dropout(classifier_dropout) + self.out_proj = nn.Linear(config.hidden_size, config.num_labels) + + def forward(self, features, **kwargs): + x = features[:, 0, :] # take token (equiv. to [CLS]) + x = self.dropout(x) + x = self.dense(x) + x = torch.tanh(x) + x = self.dropout(x) + x = self.out_proj(x) + return x + + +@add_start_docstrings( + """ + MEGA Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear + layers on top of the hidden-states output to compute `span start logits` and `span end logits`). + """, + MEGA_START_DOCSTRING, +) +class MegaForQuestionAnswering(MegaPreTrainedModel): + _keys_to_ignore_on_load_unexpected = [r"pooler"] + _keys_to_ignore_on_load_missing = [] + + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.mega = MegaModel(config, add_pooling_layer=False) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(MEGA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=QuestionAnsweringModelOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + start_positions: Optional[torch.LongTensor] = None, + end_positions: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]: + r""" + start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.mega( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1).contiguous() + end_logits = end_logits.squeeze(-1).contiguous() + + total_loss = None + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions = start_positions.clamp(0, ignored_index) + end_positions = end_positions.clamp(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + + if not return_dict: + output = (start_logits, end_logits) + outputs[2:] + return ((total_loss,) + output) if total_loss is not None else output + + return QuestionAnsweringModelOutput( + loss=total_loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index b062a3bec4d9..f016538fac68 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -4205,6 +4205,65 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +MEGA_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class MegaForCausalLM(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MegaForMaskedLM(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MegaForMultipleChoice(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MegaForQuestionAnswering(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MegaForSequenceClassification(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MegaForTokenClassification(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MegaModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class MegaPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/tests/models/mega/__init__.py b/tests/models/mega/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/mega/test_modeling_mega.py b/tests/models/mega/test_modeling_mega.py new file mode 100644 index 000000000000..7ea0efb83ada --- /dev/null +++ b/tests/models/mega/test_modeling_mega.py @@ -0,0 +1,649 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +from transformers import MegaConfig, is_torch_available +from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device + +from ...generation.test_utils import GenerationTesterMixin +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask +from ...test_pipeline_mixin import PipelineTesterMixin + + +if is_torch_available(): + import torch + + from transformers import ( + MegaForCausalLM, + MegaForMaskedLM, + MegaForMultipleChoice, + MegaForQuestionAnswering, + MegaForSequenceClassification, + MegaForTokenClassification, + MegaModel, + ) + from transformers.models.mega.modeling_mega import MEGA_PRETRAINED_MODEL_ARCHIVE_LIST + + +class MegaModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + intermediate_size=37, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_positions=1024, + bidirectional=False, # needed for decoding, and can't modify common generation tests; test separately by overriding + ema_projection_size=16, + shared_representation_size=64, + use_chunking=False, + chunk_size=32, + attention_activation="softmax", + use_normalized_ffn=True, + nffn_hidden_size=24, + add_token_type_embeddings=True, + type_vocab_size=2, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.add_token_type_embeddings = add_token_type_embeddings + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_positions = max_positions + self.bidirectional = bidirectional + self.ema_projection_size = ema_projection_size + self.shared_representation_size = shared_representation_size + self.use_chunking = use_chunking + self.chunk_size = chunk_size + self.attention_activation = attention_activation + self.use_normalized_ffn = use_normalized_ffn + self.nffn_hidden_size = nffn_hidden_size + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + self.num_attention_heads = 1 + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = random_attention_mask([self.batch_size, self.seq_length]) + + token_type_ids = None + if self.add_token_type_embeddings: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = self.get_config() + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def get_config(self): + return MegaConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + intermediate_size=self.intermediate_size, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range, + # added args + add_token_type_embeddings=self.add_token_type_embeddings, + max_positions=self.max_positions, + bidirectional=self.bidirectional, + ema_projection_size=self.ema_projection_size, + shared_representation_size=self.shared_representation_size, + use_chunking=self.use_chunking, + chunk_size=self.chunk_size, + attention_activation=self.attention_activation, + use_normalized_ffn=self.use_normalized_ffn, + nffn_hidden_size=self.nffn_hidden_size, + ) + + def get_pipeline_config(self): + config = self.get_config() + config.vocab_size = 300 + return config + + def prepare_config_and_inputs_for_decoder(self): + ( + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + ) = self.prepare_config_and_inputs() + + config.is_decoder = True + config.bidirectional = False + encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]) + encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + return ( + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ) + + def create_and_check_model( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + model = MegaModel(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + result = model(input_ids, token_type_ids=token_type_ids) + result = model(input_ids) + + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) + + def create_and_check_model_as_decoder( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + config.add_cross_attention = True + model = MegaModel(config) + model.to(torch_device) + model.eval() + result = model( + input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + ) + result = model( + input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + encoder_hidden_states=encoder_hidden_states, + ) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) + + def create_and_check_for_causal_lm( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + model = MegaForCausalLM(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + + def create_and_check_decoder_model_past_large_inputs( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ): + config.is_decoder = True + config.bidirectional = False + config.add_cross_attention = True + model = MegaForCausalLM(config=config).to(torch_device).eval() + + # make sure that ids don't start with pad token + mask = input_ids.ne(config.pad_token_id).long() + input_ids = input_ids * mask + + # first forward pass + outputs = model( + input_ids, + attention_mask=input_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + use_cache=True, + ) + past_key_values = outputs.past_key_values + + # create hypothetical multiple next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size) + + # make sure that ids don't start with pad token + mask = next_tokens.ne(config.pad_token_id).long() + next_tokens = next_tokens * mask + next_mask = ids_tensor((self.batch_size, 1), vocab_size=2) + + # append to next input_ids and + next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) + next_attention_mask = torch.cat([input_mask, next_mask], dim=-1) + + output_from_no_past = model( + next_input_ids, + attention_mask=next_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_hidden_states=True, + )["hidden_states"][0] + output_from_past = model( + next_tokens, + attention_mask=next_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + output_hidden_states=True, + )["hidden_states"][0] + + # select random slice + random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() + output_from_no_past_slice = output_from_no_past[:, -1:, random_slice_idx].detach() + output_from_past_slice = output_from_past[:, :, random_slice_idx].detach() + + self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]) + + # test that outputs are equal for slice + self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) + + def create_and_check_for_masked_lm( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + model = MegaForMaskedLM(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + + def create_and_check_for_token_classification( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.num_labels = self.num_labels + model = MegaForTokenClassification(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels)) + + def create_and_check_for_multiple_choice( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.num_choices = self.num_choices + model = MegaForMultipleChoice(config=config) + model.to(torch_device) + model.eval() + multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + result = model( + multiple_choice_inputs_ids, + attention_mask=multiple_choice_input_mask, + token_type_ids=multiple_choice_token_type_ids, + labels=choice_labels, + ) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices)) + + def create_and_check_for_question_answering( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + model = MegaForQuestionAnswering(config=config) + model.to(torch_device) + model.eval() + result = model( + input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + start_positions=sequence_labels, + end_positions=sequence_labels, + ) + self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length)) + self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length)) + + # extra checks for Mega-specific model functionality + def create_and_check_bidirectionality( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.bidirectional = True + model = MegaModel(config) + model.to(torch_device) + model.eval() + # no mask + result = model(input_ids) + # with mask & token types + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + + self.parent.assertEqual(result[0].shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def check_chunking_shorter_sequence( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.use_chunking = True + config.chunk_size = input_ids.size(1) + 25 + model = MegaModel(config) + + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + + self.parent.assertEqual(result[0].shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def check_chunking_longer_sequence( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.use_chunking = True + + # we want the chunk size to be < sequence length, and the sequence length to be a multiple of chunk size + config.chunk_size = input_ids.size(1) * 2 + model = MegaModel(config) + + result = model( + input_ids.repeat(1, 8), + ) + + self.parent.assertEqual(result[0].shape, (self.batch_size, self.seq_length * 8, self.hidden_size)) + + def check_laplace_self_attention( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.attention_activation = "laplace" + model = MegaModel(config) + + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + + self.parent.assertEqual(result[0].shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def check_relu2_self_attention( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.attention_activation = "relu2" + model = MegaModel(config) + + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + + self.parent.assertEqual(result[0].shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def check_sequence_length_beyond_max_positions( + self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + ): + config.max_positions = self.seq_length - 2 + model = MegaModel(config) + + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + + self.parent.assertEqual(result[0].shape, (self.batch_size, self.seq_length, self.hidden_size)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + ( + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + ) = config_and_inputs + inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask} + return config, inputs_dict + + +@require_torch +class MegaModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): + all_model_classes = ( + ( + MegaForCausalLM, + MegaForMaskedLM, + MegaModel, + MegaForSequenceClassification, + MegaForTokenClassification, + MegaForMultipleChoice, + MegaForQuestionAnswering, + ) + if is_torch_available() + else () + ) + all_generative_model_classes = (MegaForCausalLM,) if is_torch_available() else () + pipeline_model_mapping = ( + { + "feature-extraction": MegaModel, + "question-answering": MegaForQuestionAnswering, + "text-classification": MegaForSequenceClassification, + "text-generation": MegaForCausalLM, + "zero-shot": MegaForSequenceClassification, + } + if is_torch_available() + else {} + ) + + fx_compatible = False + test_head_masking = False + test_pruning = False + + def setUp(self): + self.model_tester = MegaModelTester(self) + self.config_tester = ConfigTester(self, config_class=MegaConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_model_as_decoder(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder() + self.model_tester.create_and_check_model_as_decoder(*config_and_inputs) + + def test_model_as_decoder_with_default_input_mask(self): + # This regression test was failing with PyTorch < 1.3 + ( + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ) = self.model_tester.prepare_config_and_inputs_for_decoder() + + input_mask = None + + self.model_tester.create_and_check_model_as_decoder( + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + encoder_hidden_states, + encoder_attention_mask, + ) + + def test_for_causal_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder() + self.model_tester.create_and_check_for_causal_lm(*config_and_inputs) + + def test_decoder_model_past_with_large_inputs(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder() + self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_masked_lm(*config_and_inputs) + + def test_for_token_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_token_classification(*config_and_inputs) + + def test_for_multiple_choice(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs) + + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_question_answering(*config_and_inputs) + + def test_for_bidirectionality(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bidirectionality(*config_and_inputs) + + def test_for_chunking_shorter_sequence(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.check_chunking_shorter_sequence(*config_and_inputs) + + def test_for_chunking_longer_sequence(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.check_chunking_longer_sequence(*config_and_inputs) + + def test_for_laplace_attention(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.check_laplace_self_attention(*config_and_inputs) + + def test_for_relu2_attention(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.check_relu2_self_attention(*config_and_inputs) + + def test_for_sequence_length_beyond_max_positions(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.check_sequence_length_beyond_max_positions(*config_and_inputs) + + def test_generate_fp16(self): + config, input_ids, _, attention_mask, *_ = self.model_tester.prepare_config_and_inputs_for_decoder() + # attention_mask = torch.LongTensor(input_ids.ne(1)).to(torch_device) + model = MegaForCausalLM(config).eval().to(torch_device) + if torch_device == "cuda": + model.half() + model.generate(input_ids, attention_mask=attention_mask) + model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) + + def test_sequence_classification_model(self): + config, input_ids, _, attention_mask, *_ = self.model_tester.prepare_config_and_inputs() + config.num_labels = self.model_tester.num_labels + sequence_labels = ids_tensor([self.model_tester.batch_size], self.model_tester.type_sequence_label_size) + model = MegaForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + + def test_sequence_classification_model_for_multi_label(self): + config, input_ids, _, attention_mask, *_ = self.model_tester.prepare_config_and_inputs() + config.num_labels = self.model_tester.num_labels + config.problem_type = "multi_label_classification" + sequence_labels = ids_tensor( + [self.model_tester.batch_size, config.num_labels], self.model_tester.type_sequence_label_size + ).to(torch.float) + model = MegaForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=attention_mask, labels=sequence_labels) + self.assertEqual(result.logits.shape, (self.model_tester.batch_size, self.model_tester.num_labels)) + + @slow + def test_model_from_pretrained(self): + for model_name in MEGA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = MegaModel.from_pretrained(model_name) + self.assertIsNotNone(model) + + +@require_torch +class MegaModelIntegrationTest(TestCasePlus): + @slow + def test_inference_masked_lm(self): + model = MegaForMaskedLM.from_pretrained("mnaylor/mega-base-wikitext") + + input_ids = torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) + with torch.no_grad(): + output = model(input_ids)[0] + expected_shape = torch.Size((1, 11, 50265)) + self.assertEqual(output.shape, expected_shape) + # compare the actual values for a slice. + expected_slice = torch.tensor( + [[[67.8389, 10.1470, -32.7148], [-11.1655, 29.1152, 23.1304], [-3.8015, 66.0397, 29.6733]]] + ) + + self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4)) + + @slow + def test_inference_no_head(self): + model = MegaModel.from_pretrained("mnaylor/mega-base-wikitext") + + input_ids = torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) + with torch.no_grad(): + output = model(input_ids)[0] + expected_shape = torch.Size((1, 11, 128)) + self.assertEqual(output.shape, expected_shape) + # compare the actual values for a slice. taken from output[:, :3, :3] + expected_slice = torch.tensor( + [[[1.1767, -0.6349, 2.8494], [-0.5109, -0.7745, 1.9495], [-0.3287, -0.2111, 3.3367]]] + ) + + self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4)) From 5543b2ed6c4e3bd52f66344df1f40df7c4cd3a1e Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 24 Mar 2023 14:29:05 +0100 Subject: [PATCH 185/277] Update docker files to use official torch 2.0.0 (#22357) * update docker files to use official torch 2.0.0 --------- Co-authored-by: ydshieh --- docker/transformers-all-latest-gpu/Dockerfile | 20 +++++++++---------- .../Dockerfile | 6 ++---- docker/transformers-pytorch-gpu/Dockerfile | 8 +++----- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/docker/transformers-all-latest-gpu/Dockerfile b/docker/transformers-all-latest-gpu/Dockerfile index 9f30f4531b0a..20240825e038 100644 --- a/docker/transformers-all-latest-gpu/Dockerfile +++ b/docker/transformers-all-latest-gpu/Dockerfile @@ -24,15 +24,13 @@ ARG REF=main RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] -## TODO: Handle these in a python utility script -#RUN [ ${#PYTORCH} -gt 0 -a "$PYTORCH" != "pre" ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; echo "export VERSION='$VERSION'" >> ~/.profile -#RUN echo torch=$VERSION -## `torchvision` and `torchaudio` should be installed along with `torch`, especially for nightly build. -## Currently, let's just use their latest releases (when `torch` is installed with a release version) -## TODO: We might need to specify proper versions that work with a specific torch version (especially for past CI). -#RUN [ "$PYTORCH" != "pre" ] && python3 -m pip install --no-cache-dir -U $VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA || python3 -m pip install --no-cache-dir -U --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/$CUDA - -RUN python3 -m pip install --no-cache-dir -U torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 +# TODO: Handle these in a python utility script +RUN [ ${#PYTORCH} -gt 0 -a "$PYTORCH" != "pre" ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; echo "export VERSION='$VERSION'" >> ~/.profile +RUN echo torch=$VERSION +# `torchvision` and `torchaudio` should be installed along with `torch`, especially for nightly build. +# Currently, let's just use their latest releases (when `torch` is installed with a release version) +# TODO: We might need to specify proper versions that work with a specific torch version (especially for past CI). +RUN [ "$PYTORCH" != "pre" ] && python3 -m pip install --no-cache-dir -U $VERSION torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA || python3 -m pip install --no-cache-dir -U --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/$CUDA RUN python3 -m pip install --no-cache-dir -U tensorflow==2.11 RUN python3 -m pip install --no-cache-dir -U tensorflow_probability @@ -56,8 +54,8 @@ RUN python3 -m pip install --no-cache-dir bitsandbytes # For video model testing RUN python3 -m pip install --no-cache-dir decord av==9.2.0 -## For `dinat` model -#RUN python3 -m pip install --no-cache-dir natten -f https://shi-labs.com/natten/wheels/$CUDA/ +# For `dinat` model +RUN python3 -m pip install --no-cache-dir natten -f https://shi-labs.com/natten/wheels/$CUDA/ # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile index 2fa1317f8275..56c3c707240d 100644 --- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile +++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive -ARG PYTORCH='1.13.1' +ARG PYTORCH='2.0.0' # Example: `cu102`, `cu113`, etc. ARG CUDA='cu117' @@ -18,9 +18,7 @@ RUN git clone https://github.com/huggingface/transformers && cd transformers && # Install latest release PyTorch # (PyTorch must be installed before pre-compiling any DeepSpeed c++/cuda ops.) # (https://www.deepspeed.ai/tutorials/advanced-install/#pre-install-deepspeed-ops) -#RUN python3 -m pip install --no-cache-dir -U torch==$PYTORCH torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA -RUN python3 -m pip install --no-cache-dir -U torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 - +RUN python3 -m pip install --no-cache-dir -U torch==$PYTORCH torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/$CUDA RUN python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing] diff --git a/docker/transformers-pytorch-gpu/Dockerfile b/docker/transformers-pytorch-gpu/Dockerfile index 24564aca6fae..3cb8625339ba 100644 --- a/docker/transformers-pytorch-gpu/Dockerfile +++ b/docker/transformers-pytorch-gpu/Dockerfile @@ -18,11 +18,9 @@ ARG TORCH_AUDIO='' # Example: `cu102`, `cu113`, etc. ARG CUDA='cu117' -#RUN [ ${#PYTORCH} -gt 0 ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA -#RUN [ ${#TORCH_VISION} -gt 0 ] && VERSION='torchvision=='TORCH_VISION'.*' || VERSION='torchvision'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA -#RUN [ ${#TORCH_AUDIO} -gt 0 ] && VERSION='torchaudio=='TORCH_AUDIO'.*' || VERSION='torchaudio'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA - -RUN python3 -m pip install --no-cache-dir -U torch torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu117 +RUN [ ${#PYTORCH} -gt 0 ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA +RUN [ ${#TORCH_VISION} -gt 0 ] && VERSION='torchvision=='TORCH_VISION'.*' || VERSION='torchvision'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA +RUN [ ${#TORCH_AUDIO} -gt 0 ] && VERSION='torchaudio=='TORCH_AUDIO'.*' || VERSION='torchaudio'; python3 -m pip install --no-cache-dir -U $VERSION --extra-index-url https://download.pytorch.org/whl/$CUDA RUN python3 -m pip uninstall -y tensorflow flax From b43dac843350fd0e26d6265408fdfcaac5552acc Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:54:06 -0400 Subject: [PATCH 186/277] Pin tensorflow-text to go with tensorflow (#22362) * Pin tensorflow-text to go with tensorflow * Make it more convenient to pin TensorFlow * setup don't like f-strings --- examples/tensorflow/_tests_requirements.txt | 2 +- setup.py | 3 ++- src/transformers/dependency_versions_table.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/tensorflow/_tests_requirements.txt b/examples/tensorflow/_tests_requirements.txt index 837ce6d0d16d..4548479bcc97 100644 --- a/examples/tensorflow/_tests_requirements.txt +++ b/examples/tensorflow/_tests_requirements.txt @@ -1,4 +1,4 @@ -tensorflow<2.11 +tensorflow<2.12 tensorboard scikit-learn seqeval diff --git a/setup.py b/setup.py index 669c8d959caf..803535ab53d9 100644 --- a/setup.py +++ b/setup.py @@ -164,9 +164,10 @@ "starlette", "sudachipy>=0.6.6", "sudachidict_core>=20220729", + # TensorFlow pin. When changing this value, update examples/tensorflow/_tests_requirements.txt accordingly "tensorflow-cpu>=2.4,<2.12", "tensorflow>=2.4,<2.12", - "tensorflow-text", + "tensorflow-text<2.12", "tf2onnx", "timeout-decorator", "timm", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 7c596c266830..1fbddb013fb8 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -72,7 +72,7 @@ "sudachidict_core": "sudachidict_core>=20220729", "tensorflow-cpu": "tensorflow-cpu>=2.4,<2.12", "tensorflow": "tensorflow>=2.4,<2.12", - "tensorflow-text": "tensorflow-text", + "tensorflow-text": "tensorflow-text<2.12", "tf2onnx": "tf2onnx", "timeout-decorator": "timeout-decorator", "timm": "timm", From 72277c228552dae2f9bf42fea5de6ae8688386cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20Bub=C3=A1n?= <43829749+Mahrkeenerh@users.noreply.github.com> Date: Fri, 24 Mar 2023 19:09:01 +0100 Subject: [PATCH 187/277] Improve error message (#22361) * Improve error message * Fix consistency --- src/transformers/models/vit/modeling_vit.py | 1 + src/transformers/models/vit_msn/modeling_vit_msn.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py index eba8278eaafe..b006e70814fc 100644 --- a/src/transformers/models/vit/modeling_vit.py +++ b/src/transformers/models/vit/modeling_vit.py @@ -170,6 +170,7 @@ def forward(self, pixel_values: torch.Tensor, interpolate_pos_encoding: bool = F if num_channels != self.num_channels: raise ValueError( "Make sure that the channel dimension of the pixel values match with the one set in the configuration." + f" Expected {self.num_channels} but got {num_channels}." ) if not interpolate_pos_encoding: if height != self.image_size[0] or width != self.image_size[1]: diff --git a/src/transformers/models/vit_msn/modeling_vit_msn.py b/src/transformers/models/vit_msn/modeling_vit_msn.py index 8cba7f5a524f..46639e7d622c 100644 --- a/src/transformers/models/vit_msn/modeling_vit_msn.py +++ b/src/transformers/models/vit_msn/modeling_vit_msn.py @@ -153,6 +153,7 @@ def forward(self, pixel_values: torch.Tensor, interpolate_pos_encoding: bool = F if num_channels != self.num_channels: raise ValueError( "Make sure that the channel dimension of the pixel values match with the one set in the configuration." + f" Expected {self.num_channels} but got {num_channels}." ) if not interpolate_pos_encoding: if height != self.image_size[0] or width != self.image_size[1]: From 6f7580b40c1463b70d6dbd6806ac1fe9467462c9 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Fri, 24 Mar 2023 18:45:03 +0000 Subject: [PATCH 188/277] TensorFlow: pin maximum version to 2.12 (#22364) --- .circleci/create_circleci_config.py | 2 +- examples/tensorflow/_tests_requirements.txt | 2 +- setup.py | 6 +++--- src/transformers/dependency_versions_table.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 0b26762b080a..27252284328f 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -224,7 +224,7 @@ def job_name(self): tf_job = CircleCIJob( "tf", install_steps=[ - "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake", "pip install --upgrade pip", "pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]", "pip install tensorflow_probability", diff --git a/examples/tensorflow/_tests_requirements.txt b/examples/tensorflow/_tests_requirements.txt index 4548479bcc97..02499f9321c9 100644 --- a/examples/tensorflow/_tests_requirements.txt +++ b/examples/tensorflow/_tests_requirements.txt @@ -1,4 +1,4 @@ -tensorflow<2.12 +tensorflow<2.13 tensorboard scikit-learn seqeval diff --git a/setup.py b/setup.py index 803535ab53d9..33eefac65746 100644 --- a/setup.py +++ b/setup.py @@ -165,9 +165,9 @@ "sudachipy>=0.6.6", "sudachidict_core>=20220729", # TensorFlow pin. When changing this value, update examples/tensorflow/_tests_requirements.txt accordingly - "tensorflow-cpu>=2.4,<2.12", - "tensorflow>=2.4,<2.12", - "tensorflow-text<2.12", + "tensorflow-cpu>=2.4,<2.13", + "tensorflow>=2.4,<2.13", + "tensorflow-text<2.13", "tf2onnx", "timeout-decorator", "timm", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 1fbddb013fb8..314267922b1a 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -70,9 +70,9 @@ "starlette": "starlette", "sudachipy": "sudachipy>=0.6.6", "sudachidict_core": "sudachidict_core>=20220729", - "tensorflow-cpu": "tensorflow-cpu>=2.4,<2.12", - "tensorflow": "tensorflow>=2.4,<2.12", - "tensorflow-text": "tensorflow-text<2.12", + "tensorflow-cpu": "tensorflow-cpu>=2.4,<2.13", + "tensorflow": "tensorflow>=2.4,<2.13", + "tensorflow-text": "tensorflow-text<2.13", "tf2onnx": "tf2onnx", "timeout-decorator": "timeout-decorator", "timm": "timm", From 9b89cf6b4ebf5973cb16a6226196d3e8918a9dbe Mon Sep 17 00:00:00 2001 From: Shubhamai Date: Sat, 25 Mar 2023 01:15:57 +0530 Subject: [PATCH 189/277] Resnet flax (#21472) * [WIP] flax resnet * added pretrained flax models, results reproducible * Added pretrained flax models, results reproducible * working on tests * no real code change, just some comments * [flax] adding support for batch norm layers * fixing bugs related to pt+flax integration * removing loss from modeling flax output class * fixing classifier tests * fixing comments, model output * cleaning comments * review changes * review changes * Apply suggestions from code review Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * renaming Flax to PyTorch --------- Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- docs/source/de/index.mdx | 2 +- docs/source/en/index.mdx | 2 +- docs/source/en/model_doc/resnet.mdx | 10 + docs/source/es/index.mdx | 2 +- docs/source/it/index.mdx | 2 +- docs/source/ja/index.mdx | 2 +- docs/source/ko/index.mdx | 2 +- docs/source/pt/index.mdx | 2 +- src/transformers/__init__.py | 4 + src/transformers/modeling_flax_outputs.py | 58 ++ .../models/auto/modeling_flax_auto.py | 2 + src/transformers/models/resnet/__init__.py | 27 +- .../models/resnet/modeling_flax_resnet.py | 701 ++++++++++++++++++ src/transformers/utils/dummy_flax_objects.py | 21 + .../resnet/test_modeling_flax_resnet.py | 228 ++++++ 15 files changed, 1057 insertions(+), 8 deletions(-) create mode 100644 src/transformers/models/resnet/modeling_flax_resnet.py create mode 100644 tests/models/resnet/test_modeling_flax_resnet.py diff --git a/docs/source/de/index.mdx b/docs/source/de/index.mdx index c1340820b567..5983e47bfd41 100644 --- a/docs/source/de/index.mdx +++ b/docs/source/de/index.mdx @@ -285,7 +285,7 @@ Flax), PyTorch, und/oder TensorFlow haben. | Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | | RegNet | ❌ | ❌ | ✅ | ✅ | ❌ | | RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ✅ | ❌ | +| ResNet | ❌ | ❌ | ✅ | ✅ | ✅ | | RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | | RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | | RoFormer | ✅ | ✅ | ✅ | ✅ | ✅ | diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index f67483f84f03..198f3d97de39 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -377,7 +377,7 @@ Flax), PyTorch, and/or TensorFlow. | Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | | RegNet | ❌ | ❌ | ✅ | ✅ | ❌ | | RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ✅ | ❌ | +| ResNet | ❌ | ❌ | ✅ | ✅ | ✅ | | RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | | RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | | RoBERTa-PreLayerNorm | ❌ | ❌ | ✅ | ✅ | ✅ | diff --git a/docs/source/en/model_doc/resnet.mdx b/docs/source/en/model_doc/resnet.mdx index 476698e9ab9f..a34596bdd68a 100644 --- a/docs/source/en/model_doc/resnet.mdx +++ b/docs/source/en/model_doc/resnet.mdx @@ -71,3 +71,13 @@ If you're interested in submitting a resource to be included here, please feel f [[autodoc]] TFResNetForImageClassification - call + +## FlaxResNetModel + +[[autodoc]] FlaxResNetModel + - __call__ + +## FlaxResNetForImageClassification + +[[autodoc]] FlaxResNetForImageClassification + - __call__ diff --git a/docs/source/es/index.mdx b/docs/source/es/index.mdx index baedf45a0fdc..11943daa605f 100644 --- a/docs/source/es/index.mdx +++ b/docs/source/es/index.mdx @@ -237,7 +237,7 @@ Flax), PyTorch y/o TensorFlow. | Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | | RegNet | ❌ | ❌ | ✅ | ❌ | ❌ | | RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ❌ | ❌ | +| ResNet | ❌ | ❌ | ✅ | ❌ | ✅ | | RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | | RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | | RoFormer | ✅ | ✅ | ✅ | ✅ | ✅ | diff --git a/docs/source/it/index.mdx b/docs/source/it/index.mdx index a6d474bb6c23..4ede7f0a9f91 100644 --- a/docs/source/it/index.mdx +++ b/docs/source/it/index.mdx @@ -254,7 +254,7 @@ tokenizer (chiamato "slow"). Un tokenizer "fast" supportato dalla libreria 🤗 | Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | | RegNet | ❌ | ❌ | ✅ | ❌ | ❌ | | RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ❌ | ❌ | +| ResNet | ❌ | ❌ | ✅ | ✅ | ✅ | | RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | | RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | | RoFormer | ✅ | ✅ | ✅ | ✅ | ✅ | diff --git a/docs/source/ja/index.mdx b/docs/source/ja/index.mdx index a644a885d32d..cca8f52be3ae 100644 --- a/docs/source/ja/index.mdx +++ b/docs/source/ja/index.mdx @@ -339,7 +339,7 @@ specific language governing permissions and limitations under the License. | Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | | RegNet | ❌ | ❌ | ✅ | ✅ | ❌ | | RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ✅ | ❌ | +| ResNet | ❌ | ❌ | ✅ | ✅ | ✅ | | RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | | RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | | RoBERTa-PreLayerNorm | ❌ | ❌ | ✅ | ✅ | ✅ | diff --git a/docs/source/ko/index.mdx b/docs/source/ko/index.mdx index 789aa41a28e0..4cf6863df028 100644 --- a/docs/source/ko/index.mdx +++ b/docs/source/ko/index.mdx @@ -308,7 +308,7 @@ specific language governing permissions and limitations under the License. | Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | | RegNet | ❌ | ❌ | ✅ | ✅ | ❌ | | RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ✅ | ❌ | +| ResNet | ❌ | ❌ | ✅ | ✅ | ✅ | | RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | | RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | | RoCBert | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/pt/index.mdx b/docs/source/pt/index.mdx index 8e74a1feb4fe..d20d746a2a40 100644 --- a/docs/source/pt/index.mdx +++ b/docs/source/pt/index.mdx @@ -252,7 +252,7 @@ disso, são diferenciados pelo suporte em diferentes frameworks: JAX (por meio d | Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | | RegNet | ❌ | ❌ | ✅ | ❌ | ❌ | | RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ❌ | ❌ | +| ResNet | ❌ | ❌ | ✅ | ❌ | ✅ | | RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | | RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | | RoFormer | ✅ | ✅ | ✅ | ✅ | ✅ | diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index bc32232d51d2..cffd17d428fc 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -3637,6 +3637,9 @@ "FlaxPegasusPreTrainedModel", ] ) + _import_structure["models.resnet"].extend( + ["FlaxResNetForImageClassification", "FlaxResNetModel", "FlaxResNetPreTrainedModel"] + ) _import_structure["models.roberta"].extend( [ "FlaxRobertaForCausalLM", @@ -6693,6 +6696,7 @@ from .models.mt5 import FlaxMT5EncoderModel, FlaxMT5ForConditionalGeneration, FlaxMT5Model from .models.opt import FlaxOPTForCausalLM, FlaxOPTModel, FlaxOPTPreTrainedModel from .models.pegasus import FlaxPegasusForConditionalGeneration, FlaxPegasusModel, FlaxPegasusPreTrainedModel + from .models.resnet import FlaxResNetForImageClassification, FlaxResNetModel, FlaxResNetPreTrainedModel from .models.roberta import ( FlaxRobertaForCausalLM, FlaxRobertaForMaskedLM, diff --git a/src/transformers/modeling_flax_outputs.py b/src/transformers/modeling_flax_outputs.py index 4f6cc5a901f8..179a0b787936 100644 --- a/src/transformers/modeling_flax_outputs.py +++ b/src/transformers/modeling_flax_outputs.py @@ -45,6 +45,64 @@ class FlaxBaseModelOutput(ModelOutput): attentions: Optional[Tuple[jnp.ndarray]] = None +@flax.struct.dataclass +class FlaxBaseModelOutputWithNoAttention(ModelOutput): + """ + Base class for model's outputs, with potential hidden states. + + Args: + last_hidden_state (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `jnp.ndarray` (one for the output of the embeddings, if the model has an embedding layer, + one + for the output of each layer) of shape `(batch_size, num_channels, height, width)`. Hidden-states of the + model at the output of each layer plus the optional initial embedding outputs. + """ + + last_hidden_state: jnp.ndarray = None + hidden_states: Optional[Tuple[jnp.ndarray]] = None + + +@flax.struct.dataclass +class FlaxBaseModelOutputWithPoolingAndNoAttention(ModelOutput): + """ + Base class for model's outputs that also contains a pooling of the last hidden states. + + Args: + last_hidden_state (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`): + Sequence of hidden-states at the output of the last layer of the model. + pooler_output (`jnp.ndarray` of shape `(batch_size, hidden_size)`): + Last layer hidden-state after a pooling operation on the spatial dimensions. + hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `jnp.ndarray` (one for the output of the embeddings, if the model has an embedding layer, + one + for the output of each layer) of shape `(batch_size, num_channels, height, width)`. Hidden-states of the + model at the output of each layer plus the optional initial embedding outputs. + """ + + last_hidden_state: jnp.ndarray = None + pooler_output: jnp.ndarray = None + hidden_states: Optional[Tuple[jnp.ndarray]] = None + + +@flax.struct.dataclass +class FlaxImageClassifierOutputWithNoAttention(ModelOutput): + """ + Base class for outputs of image classification models. + + Args: + logits (`jnp.ndarray` of shape `(batch_size, config.num_labels)`): + Classification (or regression if config.num_labels==1) scores (before SoftMax). + hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when + `config.output_hidden_states=True`): + Tuple of `jnp.ndarray` (one for the output of the embeddings, if the model has an embedding layer, + one + for the output of each stage) of shape `(batch_size, num_channels, height, width)`. Hidden-states (also + called feature maps) of the model at the output of each stage. + """ + + logits: jnp.ndarray = None + hidden_states: Optional[Tuple[jnp.ndarray]] = None + + @flax.struct.dataclass class FlaxBaseModelOutputWithPast(ModelOutput): """ diff --git a/src/transformers/models/auto/modeling_flax_auto.py b/src/transformers/models/auto/modeling_flax_auto.py index 80e1a1d09ab4..c3061bbb1f7e 100644 --- a/src/transformers/models/auto/modeling_flax_auto.py +++ b/src/transformers/models/auto/modeling_flax_auto.py @@ -48,6 +48,7 @@ ("mt5", "FlaxMT5Model"), ("opt", "FlaxOPTModel"), ("pegasus", "FlaxPegasusModel"), + ("resnet", "FlaxResNetModel"), ("roberta", "FlaxRobertaModel"), ("roberta-prelayernorm", "FlaxRobertaPreLayerNormModel"), ("roformer", "FlaxRoFormerModel"), @@ -119,6 +120,7 @@ [ # Model for Image-classsification ("beit", "FlaxBeitForImageClassification"), + ("resnet", "FlaxResNetForImageClassification"), ("vit", "FlaxViTForImageClassification"), ] ) diff --git a/src/transformers/models/resnet/__init__.py b/src/transformers/models/resnet/__init__.py index 2baa71a277fb..62e6b1c2ca1a 100644 --- a/src/transformers/models/resnet/__init__.py +++ b/src/transformers/models/resnet/__init__.py @@ -13,7 +13,13 @@ # limitations under the License. from typing import TYPE_CHECKING -from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available +from ...utils import ( + OptionalDependencyNotAvailable, + _LazyModule, + is_flax_available, + is_tf_available, + is_torch_available, +) _import_structure = { @@ -47,6 +53,17 @@ "TFResNetPreTrainedModel", ] +try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_flax_resnet"] = [ + "FlaxResNetForImageClassification", + "FlaxResNetModel", + "FlaxResNetPreTrainedModel", + ] if TYPE_CHECKING: from .configuration_resnet import RESNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ResNetConfig, ResNetOnnxConfig @@ -78,6 +95,14 @@ TFResNetPreTrainedModel, ) + try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_flax_resnet import FlaxResNetForImageClassification, FlaxResNetModel, FlaxResNetPreTrainedModel + else: import sys diff --git a/src/transformers/models/resnet/modeling_flax_resnet.py b/src/transformers/models/resnet/modeling_flax_resnet.py new file mode 100644 index 000000000000..36b286960743 --- /dev/null +++ b/src/transformers/models/resnet/modeling_flax_resnet.py @@ -0,0 +1,701 @@ +# coding=utf-8 +# Copyright 2023 HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partial +from typing import Optional, Tuple + +import flax.linen as nn +import jax +import jax.numpy as jnp +from flax.core.frozen_dict import FrozenDict, freeze, unfreeze +from flax.traverse_util import flatten_dict, unflatten_dict + +from ...modeling_flax_outputs import ( + FlaxBaseModelOutputWithNoAttention, + FlaxBaseModelOutputWithPoolingAndNoAttention, + FlaxImageClassifierOutputWithNoAttention, +) +from ...modeling_flax_utils import ( + ACT2FN, + FlaxPreTrainedModel, + append_replace_return_docstrings, + overwrite_call_docstring, +) +from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward +from .configuration_resnet import ResNetConfig + + +RESNET_START_DOCSTRING = r""" + + This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading, saving and converting weights from PyTorch models) + + This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) + subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to + general usage and behavior. + + Finally, this model supports inherent JAX features such as: + + - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) + - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) + - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) + - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) + + Parameters: + config ([`ResNetConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights. + dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`): + The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and + `jax.numpy.bfloat16` (on TPUs). + + This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If + specified all the computation will be performed with the given `dtype`. + + **Note that this only specifies the dtype of the computation and does not influence the dtype of model + parameters.** + + If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and + [`~FlaxPreTrainedModel.to_bf16`]. +""" + + +RESNET_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`jax.numpy.float32` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See + [`AutoImageProcessor.__call__`] for details. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +class Identity(nn.Module): + """Identity function.""" + + @nn.compact + def __call__(self, x): + return x + + +class FlaxResNetConvLayer(nn.Module): + out_channels: int + kernel_size: int = 3 + stride: int = 1 + activation: Optional[str] = "relu" + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.convolution = nn.Conv( + self.out_channels, + kernel_size=(self.kernel_size, self.kernel_size), + strides=self.stride, + padding=self.kernel_size // 2, + dtype=self.dtype, + use_bias=False, + kernel_init=nn.initializers.variance_scaling(2.0, mode="fan_out", distribution="normal", dtype=self.dtype), + ) + self.normalization = nn.BatchNorm(momentum=0.9, epsilon=1e-05, dtype=self.dtype) + self.activation_func = ACT2FN[self.activation] if self.activation is not None else Identity() + + def __call__(self, x: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + hidden_state = self.convolution(x) + hidden_state = self.normalization(hidden_state, use_running_average=deterministic) + hidden_state = self.activation_func(hidden_state) + return hidden_state + + +class FlaxResNetEmbeddings(nn.Module): + """ + ResNet Embeddings (stem) composed of a single aggressive convolution. + """ + + config: ResNetConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.embedder = FlaxResNetConvLayer( + self.config.embedding_size, + kernel_size=7, + stride=2, + activation=self.config.hidden_act, + dtype=self.dtype, + ) + + self.max_pool = partial(nn.max_pool, window_shape=(3, 3), strides=(2, 2), padding=((1, 1), (1, 1))) + + def __call__(self, pixel_values: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + num_channels = pixel_values.shape[-1] + if num_channels != self.config.num_channels: + raise ValueError( + "Make sure that the channel dimension of the pixel values match with the one set in the configuration." + ) + embedding = self.embedder(pixel_values, deterministic=deterministic) + embedding = self.max_pool(embedding) + return embedding + + +class FlaxResNetShortCut(nn.Module): + """ + ResNet shortcut, used to project the residual features to the correct size. If needed, it is also used to + downsample the input using `stride=2`. + """ + + out_channels: int + stride: int = 2 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.convolution = nn.Conv( + self.out_channels, + kernel_size=(1, 1), + strides=self.stride, + use_bias=False, + kernel_init=nn.initializers.variance_scaling(2.0, mode="fan_out", distribution="truncated_normal"), + dtype=self.dtype, + ) + self.normalization = nn.BatchNorm(momentum=0.9, epsilon=1e-05, dtype=self.dtype) + + def __call__(self, x: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + hidden_state = self.convolution(x) + hidden_state = self.normalization(hidden_state, use_running_average=deterministic) + return hidden_state + + +class FlaxResNetBasicLayerCollection(nn.Module): + out_channels: int + stride: int = 1 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.layer = [ + FlaxResNetConvLayer(self.out_channels, stride=self.stride, dtype=self.dtype), + FlaxResNetConvLayer(self.out_channels, activation=None, dtype=self.dtype), + ] + + def __call__(self, hidden_state: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + for layer in self.layer: + hidden_state = layer(hidden_state, deterministic=deterministic) + return hidden_state + + +class FlaxResNetBasicLayer(nn.Module): + """ + A classic ResNet's residual layer composed by two `3x3` convolutions. + """ + + in_channels: int + out_channels: int + stride: int = 1 + activation: Optional[str] = "relu" + dtype: jnp.dtype = jnp.float32 + + def setup(self): + should_apply_shortcut = self.in_channels != self.out_channels or self.stride != 1 + self.shortcut = ( + FlaxResNetShortCut(self.out_channels, stride=self.stride, dtype=self.dtype) + if should_apply_shortcut + else None + ) + self.layer = FlaxResNetBasicLayerCollection( + out_channels=self.out_channels, + stride=self.stride, + activation=self.activation, + dtype=self.dtype, + ) + self.activation_func = ACT2FN[self.activation] + + def __call__(self, hidden_state, deterministic: bool = True): + residual = hidden_state + hidden_state = self.layer(hidden_state, deterministic=deterministic) + + if self.shortcut is not None: + residual = self.shortcut(residual, deterministic=deterministic) + hidden_state += residual + + hidden_state = self.activation_func(hidden_state) + return hidden_state + + +class FlaxResNetBottleNeckLayerCollection(nn.Module): + out_channels: int + stride: int = 1 + activation: Optional[str] = "relu" + reduction: int = 4 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + reduces_channels = self.out_channels // self.reduction + + self.layer = [ + FlaxResNetConvLayer(reduces_channels, kernel_size=1, dtype=self.dtype, name="0"), + FlaxResNetConvLayer(reduces_channels, stride=self.stride, dtype=self.dtype, name="1"), + FlaxResNetConvLayer(self.out_channels, kernel_size=1, activation=None, dtype=self.dtype, name="2"), + ] + + def __call__(self, hidden_state: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + for layer in self.layer: + hidden_state = layer(hidden_state, deterministic=deterministic) + return hidden_state + + +class FlaxResNetBottleNeckLayer(nn.Module): + """ + A classic ResNet's bottleneck layer composed by three `3x3` convolutions. The first `1x1` convolution reduces the + input by a factor of `reduction` in order to make the second `3x3` convolution faster. The last `1x1` convolution + remaps the reduced features to `out_channels`. + """ + + in_channels: int + out_channels: int + stride: int = 1 + activation: Optional[str] = "relu" + reduction: int = 4 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + should_apply_shortcut = self.in_channels != self.out_channels or self.stride != 1 + self.shortcut = ( + FlaxResNetShortCut(self.out_channels, stride=self.stride, dtype=self.dtype) + if should_apply_shortcut + else None + ) + + self.layer = FlaxResNetBottleNeckLayerCollection( + self.out_channels, + stride=self.stride, + activation=self.activation, + reduction=self.reduction, + dtype=self.dtype, + ) + + self.activation_func = ACT2FN[self.activation] + + def __call__(self, hidden_state: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + residual = hidden_state + + if self.shortcut is not None: + residual = self.shortcut(residual, deterministic=deterministic) + hidden_state = self.layer(hidden_state, deterministic) + hidden_state += residual + hidden_state = self.activation_func(hidden_state) + return hidden_state + + +class FlaxResNetStageLayersCollection(nn.Module): + """ + A ResNet stage composed by stacked layers. + """ + + config: ResNetConfig + in_channels: int + out_channels: int + stride: int = 2 + depth: int = 2 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + layer = FlaxResNetBottleNeckLayer if self.config.layer_type == "bottleneck" else FlaxResNetBasicLayer + + layers = [ + # downsampling is done in the first layer with stride of 2 + layer( + self.in_channels, + self.out_channels, + stride=self.stride, + activation=self.config.hidden_act, + dtype=self.dtype, + name="0", + ), + ] + + for i in range(self.depth - 1): + layers.append( + layer( + self.out_channels, + self.out_channels, + activation=self.config.hidden_act, + dtype=self.dtype, + name=str(i + 1), + ) + ) + + self.layers = layers + + def __call__(self, x: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + hidden_state = x + for layer in self.layers: + hidden_state = layer(hidden_state, deterministic=deterministic) + return hidden_state + + +class FlaxResNetStage(nn.Module): + """ + A ResNet stage composed by stacked layers. + """ + + config: ResNetConfig + in_channels: int + out_channels: int + stride: int = 2 + depth: int = 2 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.layers = FlaxResNetStageLayersCollection( + self.config, + in_channels=self.in_channels, + out_channels=self.out_channels, + stride=self.stride, + depth=self.depth, + dtype=self.dtype, + ) + + def __call__(self, x: jnp.ndarray, deterministic: bool = True) -> jnp.ndarray: + return self.layers(x, deterministic=deterministic) + + +class FlaxResNetStageCollection(nn.Module): + config: ResNetConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + in_out_channels = zip(self.config.hidden_sizes, self.config.hidden_sizes[1:]) + stages = [ + FlaxResNetStage( + self.config, + self.config.embedding_size, + self.config.hidden_sizes[0], + stride=2 if self.config.downsample_in_first_stage else 1, + depth=self.config.depths[0], + dtype=self.dtype, + name="0", + ) + ] + + for i, ((in_channels, out_channels), depth) in enumerate(zip(in_out_channels, self.config.depths[1:])): + stages.append( + FlaxResNetStage(self.config, in_channels, out_channels, depth=depth, dtype=self.dtype, name=str(i + 1)) + ) + + self.stages = stages + + def __call__( + self, + hidden_state: jnp.ndarray, + output_hidden_states: bool = False, + deterministic: bool = True, + ) -> FlaxBaseModelOutputWithNoAttention: + hidden_states = () if output_hidden_states else None + + for stage_module in self.stages: + if output_hidden_states: + hidden_states = hidden_states + (hidden_state.transpose(0, 3, 1, 2),) + + hidden_state = stage_module(hidden_state, deterministic=deterministic) + + return hidden_state, hidden_states + + +class FlaxResNetEncoder(nn.Module): + config: ResNetConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.stages = FlaxResNetStageCollection(self.config, dtype=self.dtype) + + def __call__( + self, + hidden_state: jnp.ndarray, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ) -> FlaxBaseModelOutputWithNoAttention: + hidden_state, hidden_states = self.stages( + hidden_state, output_hidden_states=output_hidden_states, deterministic=deterministic + ) + + if output_hidden_states: + hidden_states = hidden_states + (hidden_state.transpose(0, 3, 1, 2),) + + if not return_dict: + return tuple(v for v in [hidden_state, hidden_states] if v is not None) + + return FlaxBaseModelOutputWithNoAttention( + last_hidden_state=hidden_state, + hidden_states=hidden_states, + ) + + +class FlaxResNetPreTrainedModel(FlaxPreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = ResNetConfig + base_model_prefix = "resnet" + main_input_name = "pixel_values" + module_class: nn.Module = None + + def __init__( + self, + config: ResNetConfig, + input_shape=(1, 224, 224, 3), + seed: int = 0, + dtype: jnp.dtype = jnp.float32, + _do_init: bool = True, + **kwargs, + ): + module = self.module_class(config=config, dtype=dtype, **kwargs) + if input_shape is None: + input_shape = (1, config.image_size, config.image_size, config.num_channels) + super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype, _do_init=_do_init) + + def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: + # init input tensors + pixel_values = jnp.zeros(input_shape, dtype=self.dtype) + + rngs = {"params": rng} + + random_params = self.module.init(rngs, pixel_values, return_dict=False) + + if params is not None: + random_params = flatten_dict(unfreeze(random_params)) + params = flatten_dict(unfreeze(params)) + for missing_key in self._missing_keys: + params[missing_key] = random_params[missing_key] + self._missing_keys = set() + return freeze(unflatten_dict(params)) + else: + return random_params + + @add_start_docstrings_to_model_forward(RESNET_INPUTS_DOCSTRING) + def __call__( + self, + pixel_values, + params: dict = None, + train: bool = False, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + pixel_values = jnp.transpose(pixel_values, (0, 2, 3, 1)) + + # Handle any PRNG if needed + rngs = {} + + return self.module.apply( + { + "params": params["params"] if params is not None else self.params["params"], + "batch_stats": params["batch_stats"] if params is not None else self.params["batch_stats"], + }, + jnp.array(pixel_values, dtype=jnp.float32), + not train, + output_hidden_states, + return_dict, + rngs=rngs, + mutable=["batch_stats"] if train else False, # Returing tuple with batch_stats only when train is True + ) + + +class FlaxResNetModule(nn.Module): + config: ResNetConfig + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + + def setup(self): + self.embedder = FlaxResNetEmbeddings(self.config, dtype=self.dtype) + self.encoder = FlaxResNetEncoder(self.config, dtype=self.dtype) + + # Adaptive average pooling used in resnet + self.pooler = partial( + nn.avg_pool, + padding=((0, 0), (0, 0)), + ) + + def __call__( + self, + pixel_values, + deterministic: bool = True, + output_hidden_states: bool = False, + return_dict: bool = True, + ) -> FlaxBaseModelOutputWithPoolingAndNoAttention: + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + embedding_output = self.embedder(pixel_values, deterministic=deterministic) + + encoder_outputs = self.encoder( + embedding_output, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=deterministic, + ) + + last_hidden_state = encoder_outputs[0] + + pooled_output = self.pooler( + last_hidden_state, + window_shape=(last_hidden_state.shape[1], last_hidden_state.shape[2]), + strides=(last_hidden_state.shape[1], last_hidden_state.shape[2]), + ).transpose(0, 3, 1, 2) + + last_hidden_state = last_hidden_state.transpose(0, 3, 1, 2) + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return FlaxBaseModelOutputWithPoolingAndNoAttention( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + ) + + +@add_start_docstrings( + "The bare ResNet model outputting raw features without any specific head on top.", + RESNET_START_DOCSTRING, +) +class FlaxResNetModel(FlaxResNetPreTrainedModel): + module_class = FlaxResNetModule + + +FLAX_VISION_MODEL_DOCSTRING = """ + Returns: + + Examples: + + ```python + >>> from transformers import AutoImageProcessor, FlaxResNetModel + >>> from PIL import Image + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/resnet-50") + >>> model = FlaxResNetModel.from_pretrained("microsoft/resnet-50") + >>> inputs = image_processor(images=image, return_tensors="np") + >>> outputs = model(**inputs) + >>> last_hidden_states = outputs.last_hidden_state + ``` +""" + +overwrite_call_docstring(FlaxResNetModel, FLAX_VISION_MODEL_DOCSTRING) +append_replace_return_docstrings( + FlaxResNetModel, output_type=FlaxBaseModelOutputWithPoolingAndNoAttention, config_class=ResNetConfig +) + + +class FlaxResNetClassifierCollection(nn.Module): + config: ResNetConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.classifier = nn.Dense(self.config.num_labels, dtype=self.dtype, name="1") + + def __call__(self, x: jnp.ndarray) -> jnp.ndarray: + return self.classifier(x) + + +class FlaxResNetForImageClassificationModule(nn.Module): + config: ResNetConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.resnet = FlaxResNetModule(config=self.config, dtype=self.dtype) + + if self.config.num_labels > 0: + self.classifier = FlaxResNetClassifierCollection(self.config, dtype=self.dtype) + else: + self.classifier = Identity() + + def __call__( + self, + pixel_values=None, + deterministic: bool = True, + output_hidden_states=None, + return_dict=None, + ): + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.resnet( + pixel_values, + deterministic=deterministic, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs.pooler_output if return_dict else outputs[1] + + logits = self.classifier(pooled_output[:, :, 0, 0]) + + if not return_dict: + output = (logits,) + outputs[2:] + return output + + return FlaxImageClassifierOutputWithNoAttention(logits=logits, hidden_states=outputs.hidden_states) + + +@add_start_docstrings( + """ + ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for + ImageNet. + """, + RESNET_START_DOCSTRING, +) +class FlaxResNetForImageClassification(FlaxResNetPreTrainedModel): + module_class = FlaxResNetForImageClassificationModule + + +FLAX_VISION_CLASSIF_DOCSTRING = """ + Returns: + + Example: + + ```python + >>> from transformers import AutoImageProcessor, FlaxResNetForImageClassification + >>> from PIL import Image + >>> import jax + >>> import requests + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/resnet-50") + >>> model = FlaxResNetForImageClassification.from_pretrained("microsoft/resnet-50") + + >>> inputs = image_processor(images=image, return_tensors="np") + >>> outputs = model(**inputs) + >>> logits = outputs.logits + + >>> # model predicts one of the 1000 ImageNet classes + >>> predicted_class_idx = jax.numpy.argmax(logits, axis=-1) + >>> print("Predicted class:", model.config.id2label[predicted_class_idx.item()]) + ``` +""" + +overwrite_call_docstring(FlaxResNetForImageClassification, FLAX_VISION_CLASSIF_DOCSTRING) +append_replace_return_docstrings( + FlaxResNetForImageClassification, output_type=FlaxImageClassifierOutputWithNoAttention, config_class=ResNetConfig +) diff --git a/src/transformers/utils/dummy_flax_objects.py b/src/transformers/utils/dummy_flax_objects.py index 31ce1e22d62c..c56716583994 100644 --- a/src/transformers/utils/dummy_flax_objects.py +++ b/src/transformers/utils/dummy_flax_objects.py @@ -881,6 +881,27 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) +class FlaxResNetForImageClassification(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxResNetModel(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxResNetPreTrainedModel(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + class FlaxRobertaForCausalLM(metaclass=DummyObject): _backends = ["flax"] diff --git a/tests/models/resnet/test_modeling_flax_resnet.py b/tests/models/resnet/test_modeling_flax_resnet.py new file mode 100644 index 000000000000..ee56cfe113b1 --- /dev/null +++ b/tests/models/resnet/test_modeling_flax_resnet.py @@ -0,0 +1,228 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import unittest + +from transformers import ResNetConfig, is_flax_available +from transformers.testing_utils import require_flax, slow +from transformers.utils import cached_property, is_vision_available + +from ...test_configuration_common import ConfigTester +from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor + + +if is_flax_available(): + import jax + import jax.numpy as jnp + + from transformers.models.resnet.modeling_flax_resnet import FlaxResNetForImageClassification, FlaxResNetModel + +if is_vision_available(): + from PIL import Image + + from transformers import AutoFeatureExtractor + + +class FlaxResNetModelTester(unittest.TestCase): + def __init__( + self, + parent, + batch_size=3, + image_size=32, + num_channels=3, + embeddings_size=10, + hidden_sizes=[10, 20, 30, 40], + depths=[1, 1, 2, 1], + is_training=True, + use_labels=True, + hidden_act="relu", + num_labels=3, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.image_size = image_size + self.num_channels = num_channels + self.embeddings_size = embeddings_size + self.hidden_sizes = hidden_sizes + self.depths = depths + self.is_training = is_training + self.use_labels = use_labels + self.hidden_act = hidden_act + self.num_labels = num_labels + self.scope = scope + self.num_stages = len(hidden_sizes) + + def prepare_config_and_inputs(self): + pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) + + config = self.get_config() + + return config, pixel_values + + def get_config(self): + return ResNetConfig( + num_channels=self.num_channels, + embeddings_size=self.embeddings_size, + hidden_sizes=self.hidden_sizes, + depths=self.depths, + hidden_act=self.hidden_act, + num_labels=self.num_labels, + image_size=self.image_size, + ) + + def create_and_check_model(self, config, pixel_values): + model = FlaxResNetModel(config=config) + result = model(pixel_values) + + # Output shape (b, c, h, w) + self.parent.assertEqual( + result.last_hidden_state.shape, + (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32), + ) + + def create_and_check_for_image_classification(self, config, pixel_values): + config.num_labels = self.num_labels + model = FlaxResNetForImageClassification(config=config) + result = model(pixel_values) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, pixel_values = config_and_inputs + inputs_dict = {"pixel_values": pixel_values} + return config, inputs_dict + + +@require_flax +class FlaxResNetModelTest(FlaxModelTesterMixin, unittest.TestCase): + all_model_classes = (FlaxResNetModel, FlaxResNetForImageClassification) if is_flax_available() else () + + is_encoder_decoder = False + test_head_masking = False + has_attentions = False + + def setUp(self) -> None: + self.model_tester = FlaxResNetModelTester(self) + self.config_tester = ConfigTester(self, config_class=ResNetConfig, has_text_modality=False) + + def test_config(self): + self.create_and_test_config_common_properties() + self.config_tester.create_and_test_config_to_json_string() + self.config_tester.create_and_test_config_to_json_file() + self.config_tester.create_and_test_config_from_and_save_pretrained() + self.config_tester.create_and_test_config_with_num_labels() + self.config_tester.check_config_can_be_init_without_params() + self.config_tester.check_config_arguments_init() + + def create_and_test_config_common_properties(self): + return + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_for_image_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_image_classification(*config_and_inputs) + + @unittest.skip(reason="ResNet does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="ResNet does not support input and output embeddings") + def test_model_common_attributes(self): + pass + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.__call__) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["pixel_values"] + self.assertListEqual(arg_names[:1], expected_arg_names) + + def test_hidden_states_output(self): + def check_hidden_states_output(inputs_dict, config, model_class): + model = model_class(config) + + outputs = model(**self._prepare_for_class(inputs_dict, model_class)) + + hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states + + expected_num_stages = self.model_tester.num_stages + self.assertEqual(len(hidden_states), expected_num_stages + 1) + + @unittest.skip(reason="ResNet does not use feedforward chunking") + def test_feed_forward_chunking(self): + pass + + def test_jit_compilation(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + prepared_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + model = model_class(config) + + @jax.jit + def model_jitted(pixel_values, **kwargs): + return model(pixel_values=pixel_values, **kwargs) + + with self.subTest("JIT Enabled"): + jitted_outputs = model_jitted(**prepared_inputs_dict).to_tuple() + + with self.subTest("JIT Disabled"): + with jax.disable_jit(): + outputs = model_jitted(**prepared_inputs_dict).to_tuple() + + self.assertEqual(len(outputs), len(jitted_outputs)) + for jitted_output, output in zip(jitted_outputs, outputs): + self.assertEqual(jitted_output.shape, output.shape) + + +# We will verify our results on an image of cute cats +def prepare_img(): + image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png") + return image + + +@require_flax +class FlaxResNetModelIntegrationTest(unittest.TestCase): + @cached_property + def default_feature_extractor(self): + return AutoFeatureExtractor.from_pretrained("microsoft/resnet-50") if is_vision_available() else None + + @slow + def test_inference_image_classification_head(self): + model = FlaxResNetForImageClassification.from_pretrained("microsoft/resnet-50") + + feature_extractor = self.default_feature_extractor + image = prepare_img() + inputs = feature_extractor(images=image, return_tensors="np") + + outputs = model(**inputs) + + # verify the logits + expected_shape = (1, 1000) + self.assertEqual(outputs.logits.shape, expected_shape) + + expected_slice = jnp.array([-11.1069, -9.7877, -8.3777]) + + self.assertTrue(jnp.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4)) From 71aa65144dcca0c28d403a24d567466789feee32 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 24 Mar 2023 12:46:41 -0700 Subject: [PATCH 190/277] [Trainer] add disclaimer that full_determinism is slow (#22368) --- src/transformers/training_args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index 4c13ad0af415..2a3c32673262 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -538,7 +538,7 @@ class TrainingArguments: CUDA Out-of-Memory errors. Requires accelerate to be installed (`pip install accelerate`) full_determinism (`bool`, *optional*, defaults to `False`) If `True`, [`enable_full_determinism`] is called instead of [`set_seed`] to ensure reproducible results in - distributed training + distributed training. Important: this will negatively impact the performance, so only use it for debugging. torchdynamo (`str`, *optional*): If set, the backend compiler for TorchDynamo. Possible choices are `"eager"`, `"aot_eager"`, `"inductor"`, `"nvfuser"`, `"aot_nvfuser"`, `"aot_cudagraphs"`, `"ofi"`, `"fx2trt"`, `"onnxrt"` and `"ipex"`. @@ -1055,7 +1055,7 @@ class TrainingArguments: metadata={ "help": ( "Whether to call enable_full_determinism instead of set_seed for reproducibility in distributed" - " training" + " training. Important: this will negatively impact the performance, so only use it for debugging." ) }, ) From ada526923756e8071d1a55e11c076e6d795ca36d Mon Sep 17 00:00:00 2001 From: Sylvain Gugger Date: Fri, 24 Mar 2023 16:16:43 -0400 Subject: [PATCH 191/277] Fix TF pipeline job --- .circleci/create_circleci_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 27252284328f..0f682d89582f 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -263,6 +263,7 @@ def job_name(self): "pipelines_tf", additional_env={"RUN_PIPELINE_TESTS": True}, install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y cmake", "pip install --upgrade pip", "pip install .[sklearn,tf-cpu,testing,sentencepiece,vision]", "pip install tensorflow_probability", From 7177ed1c3b5012533707f7bd6fc515a956c45fb5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Fri, 24 Mar 2023 13:23:27 -0700 Subject: [PATCH 192/277] [safetensors] don't use in `torch<1.10` (#22370) * [safetensors] don't use in pt<1.10 * better fix --- src/transformers/utils/import_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index ceb876040fcd..465dd56d3711 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -577,7 +577,13 @@ def is_optimum_available(): def is_safetensors_available(): - return importlib.util.find_spec("safetensors") is not None + if is_torch_available(): + if version.parse(_torch_version) >= version.parse("1.10"): + return importlib.util.find_spec("safetensors") is not None + else: + return False + else: + return importlib.util.find_spec("safetensors") is not None def is_tokenizers_available(): From 3252dc38eacd816635d491a285234250dac6d3d8 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Mon, 27 Mar 2023 14:20:56 +0100 Subject: [PATCH 193/277] TensorFlow: additional missing `cmake` dependencies in CI (#22383) * missing cmake * more cmake --- .circleci/create_circleci_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 0f682d89582f..a5ebdc85f66e 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -182,7 +182,7 @@ def job_name(self): "torch_and_tf", additional_env={"RUN_PT_TF_CROSS_TESTS": True}, install_steps=[ - "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs", + "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake", "git lfs install", "pip install --upgrade pip", "pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]", @@ -319,6 +319,7 @@ def job_name(self): "examples_tensorflow", cache_name="tensorflow_examples", install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y cmake", "pip install --upgrade pip", "pip install .[sklearn,tensorflow,sentencepiece,testing]", "pip install -r examples/tensorflow/_tests_requirements.txt", @@ -356,6 +357,7 @@ def job_name(self): onnx_job = CircleCIJob( "onnx", install_steps=[ + "sudo apt-get -y update && sudo apt-get install -y cmake", "pip install --upgrade pip", "pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]", ], From 2fee8e604d4630132c6bbdccd5e7c1d6111355fd Mon Sep 17 00:00:00 2001 From: Charlie-Bell <103143406+Charlie-Bell@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:24:25 +0200 Subject: [PATCH 194/277] Changed world_size() to get_world_size() bugfix (#22381) Edited one line in src/transormers/generation/utils.py. Changed dist.world_size() to dist.get_world_size() since world_size() doesn't exist in pytorch.dist. --- src/transformers/generation/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 6140f4cb4007..1ccb41292158 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -1193,7 +1193,7 @@ def generate( """ if synced_gpus is None: - if is_deepspeed_zero3_enabled() and dist.world_size() > 1: + if is_deepspeed_zero3_enabled() and dist.get_world_size() > 1: synced_gpus = True else: synced_gpus = False From 69409c54811c352d8740896099ed0960963b26cc Mon Sep 17 00:00:00 2001 From: Nicola Procopio Date: Mon, 27 Mar 2023 15:48:49 +0200 Subject: [PATCH 195/277] Translated documentation in italian (#22388) * updated toctree * added and translated mdx documents --- docs/source/it/_toctree.yml | 8 ++++++++ docs/source/it/perf_infer_gpu_many.mdx | 24 ++++++++++++++++++++++++ docs/source/it/perf_infer_special.mdx | 14 ++++++++++++++ docs/source/it/perf_train_special.mdx | 20 ++++++++++++++++++++ docs/source/it/perf_train_tpu.mdx | 20 ++++++++++++++++++++ 5 files changed, 86 insertions(+) create mode 100644 docs/source/it/perf_infer_gpu_many.mdx create mode 100644 docs/source/it/perf_infer_special.mdx create mode 100644 docs/source/it/perf_train_special.mdx create mode 100644 docs/source/it/perf_train_tpu.mdx diff --git a/docs/source/it/_toctree.yml b/docs/source/it/_toctree.yml index 630ade2212d9..47d90f9a9a85 100644 --- a/docs/source/it/_toctree.yml +++ b/docs/source/it/_toctree.yml @@ -37,10 +37,18 @@ title: Addestramento efficiente su CPU - local: perf_train_cpu_many title: Addestramento efficiente su multiple CPU + - local: perf_train_tpu + title: Addestramento su TPU + - local: perf_train_special + title: Addestramento su Hardware Specializzato - local: perf_infer_cpu title: Inferenza Efficiente su CPU - local: perf_infer_gpu_one title: Inferenza su una GPU + - local: perf_infer_gpu_many + title: Inferenza Efficiente su GPU Multiple + - local: perf_infer_special + title: Inferenza su Hardware Specializzato - local: big_models title: Istanziare un big model - local: migration diff --git a/docs/source/it/perf_infer_gpu_many.mdx b/docs/source/it/perf_infer_gpu_many.mdx new file mode 100644 index 000000000000..5eeefa907dd6 --- /dev/null +++ b/docs/source/it/perf_infer_gpu_many.mdx @@ -0,0 +1,24 @@ + + +# Inferenza Efficiente su GPU Multiple + +Questo documento contiene informazioni su come fare inferenza in maniera efficiente su GPU multiple. + + + +Nota: Un setup con GPU multiple può utilizzare la maggior parte delle strategie descritte nella [sezione con GPU singola](./perf_infer_gpu_one). Tuttavia, è necessario conoscere delle tecniche semplici che possono essere utilizzate per un risultato migliore. + + + +## `BetterTransformer` per inferenza più rapida + +Abbiamo recentemente integrato `BetterTransformer` per inferenza più rapida su multi-GPU per modelli su testo, immagini e audio. Controlla il documento con queste integrazioni [qui](https://huggingface.co/docs/optimum/bettertransformer/overview) per maggiori dettagli. diff --git a/docs/source/it/perf_infer_special.mdx b/docs/source/it/perf_infer_special.mdx new file mode 100644 index 000000000000..1e92190d192c --- /dev/null +++ b/docs/source/it/perf_infer_special.mdx @@ -0,0 +1,14 @@ + + +# Inferenza su Hardware Specializzato + +Questo documento sarà completato a breve con la documentazione per l'inferenza su hardware specializzato. Nel frattempo puoi controllare [la guida per fare inferenza sulle CPU](perf_infer_cpu). \ No newline at end of file diff --git a/docs/source/it/perf_train_special.mdx b/docs/source/it/perf_train_special.mdx new file mode 100644 index 000000000000..22ea6d73e3d6 --- /dev/null +++ b/docs/source/it/perf_train_special.mdx @@ -0,0 +1,20 @@ + + +# Addestramento su Hardware Specializzato + + + + Nota: Molte delle strategie introdotte nella [sezione sulla GPU singola](perf_train_gpu_one) (come mixed precision training o gradient accumulation) e [sezione multi-GPU](perf_train_gpu_many) sono generiche e applicabili all'addestramento di modelli in generale quindi assicurati di dargli un'occhiata prima di immergerti in questa sezione. + + + +Questo documento sarà presto completato con informazioni su come effettuare la formazione su hardware specializzato. diff --git a/docs/source/it/perf_train_tpu.mdx b/docs/source/it/perf_train_tpu.mdx new file mode 100644 index 000000000000..395caebcd066 --- /dev/null +++ b/docs/source/it/perf_train_tpu.mdx @@ -0,0 +1,20 @@ + + +# Addestramento su TPU + + + + Nota: Molte delle strategie introdotte nella [sezione sulla GPU singola](perf_train_gpu_one) (come mixed precision training o gradient accumulation) e [sezione multi-GPU](perf_train_gpu_many) sono generiche e applicabili all'addestramento di modelli in generale quindi assicurati di dargli un'occhiata prima di immergerti in questa sezione. + + + +Questo documento sarà presto completato con informazioni su come effettuare la formazione su TPU. From 685e37b7ebea86dbb5c47daea4bdfd5d5a0326e0 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 27 Mar 2023 10:11:14 -0400 Subject: [PATCH 196/277] Adapt find_tied_parameters to handle breaking change in Accelerate (#22360) --- src/transformers/utils/bitsandbytes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/transformers/utils/bitsandbytes.py b/src/transformers/utils/bitsandbytes.py index aee3fec9744a..a49a620407fe 100644 --- a/src/transformers/utils/bitsandbytes.py +++ b/src/transformers/utils/bitsandbytes.py @@ -154,7 +154,12 @@ def get_keys_to_not_convert(model): tied_model = deepcopy(model) # this has 0 cost since it is done inside `init_empty_weights` context manager` tied_model.tie_weights() - tied_keys = list(find_tied_parameters(tied_model).values()) + tied_params = find_tied_parameters(tied_model) + # For compatibility with Accelerate < 0.18 + if isinstance(tied_params, dict): + tied_keys = list(tied_params.values()) + else: + tied_keys = sum([x[1:] for x in tied_params], []) has_tied_params = len(tied_keys) > 0 # Check if it is a base model From c4b9338a71d20a347027a9f95ae5bef6772b527c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BA=D1=B3=D1=B3s=D0=BD=C4=AB?= <1934337+kooshi@users.noreply.github.com> Date: Mon, 27 Mar 2023 09:34:52 -0500 Subject: [PATCH 197/277] load_in_8bit now respects 'balanced' device maps in multi-gpu environments (#22377) balanced 8bit memory --- src/transformers/modeling_utils.py | 25 ++++++++++++++----- .../models/llama/modeling_llama.py | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6245a0221158..913d7e4a9f1c 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2542,11 +2542,24 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P ) >= version.parse("0.37.0") if isinstance(device_map, str): - special_dtypes = { - name: torch.float32 - for name, _ in model.named_parameters() - if any(m in name for m in keep_in_fp32_modules) - } + special_dtypes = {} + if load_in_8bit: + special_dtypes.update( + { + name: torch_dtype + for name, _ in model.named_parameters() + if any(m in name for m in modules_to_not_convert) + } + ) + + special_dtypes.update( + { + name: torch.float32 + for name, _ in model.named_parameters() + if any(m in name for m in keep_in_fp32_modules) + } + ) + if model._no_split_modules is None: raise ValueError(f"{model.__class__.__name__} does not support `device_map='{device_map}'` yet.") no_split_modules = model._no_split_modules @@ -2569,7 +2582,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P if device_map != "sequential" and get_balanced_memory is not None: max_memory = get_balanced_memory( model, - dtype=torch_dtype, + dtype=torch_dtype if not load_in_8bit else torch.int8, low_zero=(device_map == "balanced_low_0"), max_memory=max_memory, **kwargs, diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index f069a33f8e64..f341eafb51b2 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -785,7 +785,7 @@ def forward( loss_fct = CrossEntropyLoss() shift_logits = shift_logits.view(-1, self.config.vocab_size) shift_labels = shift_labels.view(-1) - # Enable model/pipeline parallelism + # Enable model parallelism shift_labels = shift_labels.to(shift_logits.device) loss = loss_fct(shift_logits, shift_labels) From 537413816b0a586944217ace0af7a59cced510a7 Mon Sep 17 00:00:00 2001 From: Vladislav Sokolovskii <48914918+vsokolovskii@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:37:46 +0200 Subject: [PATCH 198/277] Wav2Vec2ProcessorWithLM can return N best hypotheses now (#22235) * Wav2Vec2ProcessorWithLM can return N best hypotheses now Signed-off-by: Vladislav Sokolovskii * Wav2Vec2ProcessorWithLM n_best cannot be None Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Batch decoding can return N best hypotheses now batch_decode was extended with the same functionality as decode function, N best hypotheses per sample can be returned Signed-off-by: Vladislav Sokolovskii --------- Signed-off-by: Vladislav Sokolovskii Co-authored-by: Vladislav Sokolovskii Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- .../processing_wav2vec2_with_lm.py | 98 ++++++++++++++----- 1 file changed, 76 insertions(+), 22 deletions(-) diff --git a/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py b/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py index 0f8add8b3c5d..16ce6fa035c4 100644 --- a/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py +++ b/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py @@ -50,18 +50,18 @@ class Wav2Vec2DecoderWithLMOutput(ModelOutput): text (list of `str` or `str`): Decoded logits in text from. Usually the speech transcription. logit_score (list of `float` or `float`): - Total logit score of the beam associated with produced text. + Total logit score of the beams associated with produced text. lm_score (list of `float`): - Fused lm_score of the beam associated with produced text. + Fused lm_score of the beams associated with produced text. word_offsets (list of `List[Dict[str, Union[int, str]]]` or `List[Dict[str, Union[int, str]]]`): Offsets of the decoded words. In combination with sampling rate and model downsampling rate word offsets can be used to compute time stamps for each word. """ - text: Union[List[str], str] - logit_score: Union[List[float], float] = None - lm_score: Union[List[float], float] = None - word_offsets: Union[List[ListOfDict], ListOfDict] = None + text: Union[List[List[str]], List[str], str] + logit_score: Union[List[List[float]], List[float], float] = None + lm_score: Union[List[List[float]], List[float], float] = None + word_offsets: Union[List[List[ListOfDict]], List[ListOfDict], ListOfDict] = None class Wav2Vec2ProcessorWithLM(ProcessorMixin): @@ -296,6 +296,7 @@ def batch_decode( unk_score_offset: Optional[float] = None, lm_score_boundary: Optional[bool] = None, output_word_offsets: bool = False, + n_best: int = 1, ): """ Batch decode output logits to audio transcription with language model support. @@ -350,6 +351,11 @@ def batch_decode( output_word_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output word offsets. Word offsets can be used in combination with the sampling rate and model downsampling rate to compute the time-stamps of transcribed words. + n_best (`int`, *optional*, defaults to `1`): + Number of best hypotheses to return. If `n_best` is greater than 1, the returned `text` will be a list + of lists of strings, `logit_score` will be a list of lists of floats, and `lm_score` will be a list of + lists of floats, where the length of the outer list will correspond to the batch size and the length of + the inner list will correspond to the number of returned hypotheses . The value should be >= 1. @@ -425,17 +431,40 @@ def batch_decode( # extract text and scores batch_texts, logit_scores, lm_scores, word_offsets = [], [], [], [] + for d in decoded_beams: - batch_texts.append(d[0][0]) - logit_scores.append(d[0][-2]) - lm_scores.append(d[0][-1]) - word_offsets.append([{"word": t[0], "start_offset": t[1][0], "end_offset": t[1][1]} for t in d[0][1]]) + batch_texts.append([beam[0] for beam in d]) + logit_scores.append([beam[-2] for beam in d]) + lm_scores.append([beam[-1] for beam in d]) + + # word_offsets.append([{"word": t[0], "start_offset": t[1][0], "end_offset": t[1][1]} for t in d[0][1]]) + + word_offsets.append( + [ + [ + {"word": word, "start_offset": start_offset, "end_offset": end_offset} + for word, (start_offset, end_offset) in beam[1] + ] + for beam in d + ] + ) word_offsets = word_offsets if output_word_offsets else None - return Wav2Vec2DecoderWithLMOutput( - text=batch_texts, logit_score=logit_scores, lm_score=lm_scores, word_offsets=word_offsets - ) + if n_best == 1: + return Wav2Vec2DecoderWithLMOutput( + text=[hyps[0] for hyps in batch_texts], + logit_score=[hyps[0] for hyps in logit_scores], + lm_score=[hyps[0] for hyps in lm_scores], + word_offsets=[hyps[0] for hyps in word_offsets] if word_offsets is not None else None, + ) + else: + return Wav2Vec2DecoderWithLMOutput( + text=[hyps[:n_best] for hyps in batch_texts], + logit_score=[hyps[:n_best] for hyps in logit_scores], + lm_score=[hyps[:n_best] for hyps in lm_scores], + word_offsets=[hyps[:n_best] for hyps in word_offsets] if word_offsets is not None else None, + ) def decode( self, @@ -450,6 +479,7 @@ def decode( unk_score_offset: Optional[float] = None, lm_score_boundary: Optional[bool] = None, output_word_offsets: bool = False, + n_best: int = 1, ): """ Decode output logits to audio transcription with language model support. @@ -480,6 +510,10 @@ def decode( output_word_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output word offsets. Word offsets can be used in combination with the sampling rate and model downsampling rate to compute the time-stamps of transcribed words. + n_best (`int`, *optional*, defaults to `1`): + Number of best hypotheses to return. If `n_best` is greater than 1, the returned `text` will be a list + of strings, `logit_score` will be a list of floats, and `lm_score` will be a list of floats, where the + length of these lists will correspond to the number of returned hypotheses. The value should be >= 1. @@ -564,17 +598,37 @@ def decode( word_offsets = None if output_word_offsets: word_offsets = [ - {"word": word, "start_offset": start_offset, "end_offset": end_offset} - for word, (start_offset, end_offset) in decoded_beams[0][2] + [ + {"word": word, "start_offset": start_offset, "end_offset": end_offset} + for word, (start_offset, end_offset) in beam[2] + ] + for beam in decoded_beams ] + logit_scores = [beam[-2] for beam in decoded_beams] - # more output features will be added in the future - return Wav2Vec2DecoderWithLMOutput( - text=decoded_beams[0][0], - logit_score=decoded_beams[0][-2], - lm_score=decoded_beams[0][-1], - word_offsets=word_offsets, - ) + lm_scores = [beam[-1] for beam in decoded_beams] + + hypotheses = [beam[0] for beam in decoded_beams] + + if n_best > len(decoded_beams): + logger.info( + "N-best size is larger than the number of generated hypotheses, all hypotheses will be returned." + ) + + if n_best == 1: + return Wav2Vec2DecoderWithLMOutput( + text=hypotheses[0], + logit_score=logit_scores[0], + lm_score=lm_scores[0], + word_offsets=word_offsets[0] if word_offsets is not None else None, + ) + else: + return Wav2Vec2DecoderWithLMOutput( + text=hypotheses[:n_best], + logit_score=logit_scores[:n_best], + lm_score=lm_scores[:n_best], + word_offsets=word_offsets[:n_best] if word_offsets is not None else None, + ) @contextmanager def as_target_processor(self): From 659920ad1e5deb6e5f9ac911f5d6caef72e21a48 Mon Sep 17 00:00:00 2001 From: Nathan Fradet <56734983+Natooz@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:47:35 +0200 Subject: [PATCH 199/277] Seq2seq trainer generation config arg (#22323) * seq2seq trainer and training arguments accepting GenerationConfig arg * seq2seq Trainer and training arguments docstring fixes * Update training_args_seq2seq.py docstring Co-authored-by: Joao Gante * Fixing trainer_seq2seq.py docstring Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * seq2seq trainer: legacy gen args back & GenerationConfig created at init * Seq2seq trainer: fix in case gen_config.max_new_tokens is None Co-authored-by: Joao Gante * seq2seq trainer: adding legacy arg retrocompatibility * seq2seq trainer and training arguments accepting GenerationConfig arg * seq2seq Trainer and training arguments docstring fixes * Update training_args_seq2seq.py docstring Co-authored-by: Joao Gante * Fixing trainer_seq2seq.py docstring Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * seq2seq trainer: legacy gen args back & GenerationConfig created at init * Seq2seq trainer: fix in case gen_config.max_new_tokens is None Co-authored-by: Joao Gante * seq2seq trainer: adding legacy arg retrocompatibility * seq2seq trainer: evaluate and predict untouched * Apply suggestions from code review Co-authored-by: Joao Gante * seq2seq trainer: adding init args, keeping IDEs hints --------- Co-authored-by: Joao Gante Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/trainer_seq2seq.py | 111 ++++++++++++++++++---- src/transformers/training_args_seq2seq.py | 19 +++- 2 files changed, 113 insertions(+), 17 deletions(-) diff --git a/src/transformers/trainer_seq2seq.py b/src/transformers/trainer_seq2seq.py index 3f7fb8211814..0e277f3227e0 100644 --- a/src/transformers/trainer_seq2seq.py +++ b/src/transformers/trainer_seq2seq.py @@ -12,15 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from copy import deepcopy +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import torch from torch import nn from torch.utils.data import Dataset +from .data.data_collator import DataCollator from .deepspeed import is_deepspeed_zero3_enabled +from .generation.configuration_utils import GenerationConfig +from .modeling_utils import PreTrainedModel +from .tokenization_utils_base import PreTrainedTokenizerBase from .trainer import Trainer -from .trainer_utils import PredictionOutput +from .trainer_callback import TrainerCallback +from .trainer_utils import EvalPrediction, PredictionOutput +from .training_args import TrainingArguments from .utils import logging @@ -28,6 +36,76 @@ class Seq2SeqTrainer(Trainer): + def __init__( + self, + model: Union[PreTrainedModel, nn.Module] = None, + args: TrainingArguments = None, + data_collator: Optional[DataCollator] = None, + train_dataset: Optional[Dataset] = None, + eval_dataset: Optional[Union[Dataset, Dict[str, Dataset]]] = None, + tokenizer: Optional[PreTrainedTokenizerBase] = None, + model_init: Optional[Callable[[], PreTrainedModel]] = None, + compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None, + callbacks: Optional[List[TrainerCallback]] = None, + optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None), + preprocess_logits_for_metrics: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None, + ): + super().__init__( + model=model, + args=args, + data_collator=data_collator, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + tokenizer=tokenizer, + model_init=model_init, + compute_metrics=compute_metrics, + callbacks=callbacks, + optimizers=optimizers, + preprocess_logits_for_metrics=preprocess_logits_for_metrics, + ) + + # Override self.model.generation_config if a GenerationConfig is specified in args. + # Priority: args.generation_config > model.generation_config > default GenerationConfig. + if self.args.generation_config is not None: + gen_config = self.load_generation_config(self.args.generation_config) + self.model.generation_config = gen_config + + @staticmethod + def load_generation_config(gen_config_arg: Union[str, GenerationConfig]) -> GenerationConfig: + """ + Loads a `~generation.GenerationConfig` from the `Seq2SeqTrainingArguments.generation_config` arguments. + + Args: + gen_config_arg (`str` or [`~generation.GenerationConfig`]): + `Seq2SeqTrainingArguments.generation_config` argument. + + Returns: + A `~generation.GenerationConfig`. + """ + + # GenerationConfig provided, nothing to do + if isinstance(gen_config_arg, GenerationConfig): + return deepcopy(gen_config_arg) + + # str or Path + pretrained_model_name = Path(gen_config_arg) if isinstance(gen_config_arg, str) else gen_config_arg + config_file_name = None + + # Figuring if it is path pointing to a file, pointing to a directory or else a model id or URL + # This step is required in order to determine config_file_name + if pretrained_model_name.is_file(): + config_file_name = pretrained_model_name.name + pretrained_model_name = pretrained_model_name.parent + # dir path + elif pretrained_model_name.is_dir(): + pass + # model id or URL + else: + pretrained_model_name = gen_config_arg + + gen_config = GenerationConfig.from_pretrained(pretrained_model_name, config_file_name) + return gen_config + def evaluate( self, eval_dataset: Optional[Dataset] = None, @@ -171,6 +249,8 @@ def prediction_step( inputs = self._prepare_inputs(inputs) # XXX: adapt synced_gpus for fairscale as well + # Priority (handled in generate): + # gen_kwargs > model.generation_config > default GenerationConfig() gen_kwargs = self._gen_kwargs.copy() if gen_kwargs.get("max_length") is None and gen_kwargs.get("max_new_tokens") is None: gen_kwargs["max_length"] = self.model.config.max_length @@ -192,13 +272,14 @@ def prediction_step( # removed in https://github.com/huggingface/transformers/blob/98d88b23f54e5a23e741833f1e973fdf600cc2c5/src/transformers/generation/utils.py#L1183 if self.model.generation_config._from_model_config: self.model.generation_config._from_model_config = False + + # Retrieves GenerationConfig from model.generation_config + gen_config = model.generation_config # in case the batch is shorter than max length, the output should be padded - if gen_kwargs.get("max_length") is not None and generated_tokens.shape[-1] < gen_kwargs["max_length"]: - generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_kwargs["max_length"]) - elif gen_kwargs.get("max_new_tokens") is not None and generated_tokens.shape[-1] < ( - gen_kwargs["max_new_tokens"] + 1 - ): - generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_kwargs["max_new_tokens"] + 1) + if generated_tokens.shape[-1] < gen_config.max_length: + generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_config.max_length) + elif generated_tokens.shape[-1] < gen_config.max_new_tokens + 1: + generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_config.max_new_tokens + 1) with torch.no_grad(): if has_labels: @@ -212,20 +293,18 @@ def prediction_step( loss = None if self.args.prediction_loss_only: - return (loss, None, None) + return loss, None, None if has_labels: labels = inputs["labels"] - if gen_kwargs.get("max_length") is not None and labels.shape[-1] < gen_kwargs["max_length"]: - labels = self._pad_tensors_to_max_len(labels, gen_kwargs["max_length"]) - elif gen_kwargs.get("max_new_tokens") is not None and labels.shape[-1] < ( - gen_kwargs["max_new_tokens"] + 1 - ): - labels = self._pad_tensors_to_max_len(labels, (gen_kwargs["max_new_tokens"] + 1)) + if labels.shape[-1] < gen_config.max_length: + labels = self._pad_tensors_to_max_len(labels, gen_config.max_length) + elif gen_config.max_new_tokens is not None and labels.shape[-1] < gen_config.max_new_tokens + 1: + labels = self._pad_tensors_to_max_len(labels, gen_config.max_new_tokens + 1) else: labels = None - return (loss, generated_tokens, labels) + return loss, generated_tokens, labels def _pad_tensors_to_max_len(self, tensor, max_length): if self.tokenizer is not None and hasattr(self.tokenizer, "pad_token_id"): diff --git a/src/transformers/training_args_seq2seq.py b/src/transformers/training_args_seq2seq.py index 026dce81bcfd..19e1ff1b2d26 100644 --- a/src/transformers/training_args_seq2seq.py +++ b/src/transformers/training_args_seq2seq.py @@ -14,8 +14,10 @@ import logging from dataclasses import dataclass, field -from typing import Optional +from pathlib import Path +from typing import Optional, Union +from .generation.configuration_utils import GenerationConfig from .training_args import TrainingArguments from .utils import add_start_docstrings @@ -42,6 +44,15 @@ class Seq2SeqTrainingArguments(TrainingArguments): generation_num_beams (`int`, *optional*): The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default to the `num_beams` value of the model configuration. + generation_config (`str` or `Path` or [`~generation.GenerationConfig`], *optional*): + Allows to load a [`~generation.GenerationConfig`] from the `from_pretrained` method. This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced + under a user or organization name, like `dbmdz/bert-base-german-cased`. + - a path to a *directory* containing a configuration file saved using the + [`~GenerationConfig.save_pretrained`] method, e.g., `./my_model_directory/`. + - a [`~generation.GenerationConfig`] object. """ sortish_sampler: bool = field(default=False, metadata={"help": "Whether to use SortishSampler or not."}) @@ -66,3 +77,9 @@ class Seq2SeqTrainingArguments(TrainingArguments): ) }, ) + generation_config: Optional[Union[str, Path, GenerationConfig]] = field( + default=None, + metadata={ + "help": "Model id, file path or url pointing to a GenerationConfig json file, to use during prediction." + }, + ) From 791064c2f647995dda3c8d599d6309f4e94af9bf Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Mon, 27 Mar 2023 15:48:23 +0100 Subject: [PATCH 200/277] Generate: support for left-padding on GPTNeoX and Llama (#22382) --- .../models/gpt_neox/modeling_gpt_neox.py | 80 +++++++--- src/transformers/models/gptj/modeling_gptj.py | 2 +- .../models/llama/modeling_llama.py | 148 ++++++------------ .../models/gpt_neox/test_modeling_gpt_neox.py | 2 +- 4 files changed, 108 insertions(+), 124 deletions(-) diff --git a/src/transformers/models/gpt_neox/modeling_gpt_neox.py b/src/transformers/models/gpt_neox/modeling_gpt_neox.py index 62ad98972df0..cb132e721a0a 100755 --- a/src/transformers/models/gpt_neox/modeling_gpt_neox.py +++ b/src/transformers/models/gpt_neox/modeling_gpt_neox.py @@ -100,12 +100,13 @@ def __init__(self, config): def forward( self, - hidden_states, - attention_mask, - head_mask=None, - layer_past=None, - use_cache=False, - output_attentions=False, + hidden_states: torch.FloatTensor, + attention_mask: torch.FloatTensor, + position_ids: torch.LongTensor, + head_mask: Optional[torch.FloatTensor] = None, + layer_past: Optional[Tuple[torch.Tensor]] = None, + use_cache: Optional[bool] = False, + output_attentions: Optional[bool] = False, ): has_layer_past = layer_past is not None @@ -132,12 +133,10 @@ def forward( # Compute token offset for rotary embeddings (when decoding) seq_len = key.shape[-2] - offset = 0 if has_layer_past: - offset = layer_past[0].shape[-2] - seq_len += offset + seq_len += layer_past[0].shape[-2] cos, sin = self.rotary_emb(value, seq_len=seq_len) - query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, offset=offset) + query, key = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) query = torch.cat((query, query_pass), dim=-1) key = torch.cat((key, key_pass), dim=-1) @@ -275,9 +274,11 @@ def rotate_half(x): return torch.cat((-x2, x1), dim=-1) -def apply_rotary_pos_emb(q, k, cos, sin, offset: int = 0): - cos = cos[..., offset : q.shape[-2] + offset, :] - sin = sin[..., offset : q.shape[-2] + offset, :] +def apply_rotary_pos_emb(q, k, cos, sin, position_ids): + gather_indices = position_ids[:, None, :, None] # [bs, 1, seq_len, 1] + gather_indices = gather_indices.repeat(1, cos.shape[1], 1, cos.shape[3]) + cos = torch.gather(cos.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices) + sin = torch.gather(sin.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices) q_embed = (q * cos) + (rotate_half(q) * sin) k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed @@ -308,16 +309,18 @@ def __init__(self, config): def forward( self, - hidden_states, - attention_mask=None, - head_mask=None, - use_cache=False, - layer_past=None, - output_attentions=False, + hidden_states: Optional[torch.FloatTensor], + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = False, + layer_past: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, ): attention_layer_outputs = self.attention( self.input_layernorm(hidden_states), attention_mask=attention_mask, + position_ids=position_ids, layer_past=layer_past, head_mask=head_mask, use_cache=use_cache, @@ -374,6 +377,11 @@ def forward( - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) + position_ids (`torch.LongTensor` of shape `({0})`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: @@ -430,6 +438,7 @@ def forward( self, input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, head_mask: Optional[torch.FloatTensor] = None, inputs_embeds: Optional[torch.FloatTensor] = None, past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, @@ -467,7 +476,17 @@ def forward( batch_size, seq_length = input_shape if past_key_values is None: + past_length = 0 past_key_values = tuple([None] * self.config.num_hidden_layers) + else: + past_length = past_key_values[0][0].size(-2) + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange(past_length, seq_length + past_length, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() # Attention mask. if attention_mask is not None: @@ -527,12 +546,14 @@ def custom_forward(*inputs): create_custom_forward(layer), hidden_states, attention_mask, + position_ids, head_mask[i], ) else: outputs = layer( hidden_states, attention_mask=attention_mask, + position_ids=position_ids, head_mask=head_mask[i], layer_past=layer_past, use_cache=use_cache, @@ -587,6 +608,7 @@ def forward( self, input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, inputs_embeds: Optional[torch.FloatTensor] = None, head_mask: Optional[torch.FloatTensor] = None, past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, @@ -640,6 +662,7 @@ def forward( outputs = self.gpt_neox( input_ids, attention_mask=attention_mask, + position_ids=position_ids, head_mask=head_mask, inputs_embeds=inputs_embeds, past_key_values=past_key_values, @@ -672,20 +695,29 @@ def forward( attentions=outputs.attentions, ) - def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attention_mask=None, **model_kwargs): + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attention_mask=None, **kwargs): input_shape = input_ids.shape - # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly - if attention_mask is None: - attention_mask = input_ids.new_ones(input_shape) - # cut decoder_input_ids if past is used if past_key_values and past_key_values[0] is not None: input_ids = input_ids[:, -1:] + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) + + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + return { "input_ids": input_ids, "attention_mask": attention_mask, + "position_ids": position_ids, "past_key_values": past_key_values, } diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index 6cd756bc1a33..c5e6cc564f09 100755 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -192,7 +192,7 @@ def _get_embed_positions(self, position_ids): def forward( self, - hidden_states: Optional[torch.FloatTensor], + hidden_states: torch.FloatTensor, layer_past: Optional[Tuple[torch.Tensor]] = None, attention_mask: Optional[torch.FloatTensor] = None, position_ids: Optional[torch.LongTensor] = None, diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index f341eafb51b2..cc9e01e4aa5a 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -38,6 +38,7 @@ _CONFIG_FOR_DOC = "LlamaConfig" +# Copied from transformers.models.bart.modeling_bart._make_causal_mask def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): """ Make causal mask used for bi-directional self-attention. @@ -53,6 +54,7 @@ def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_ return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) +# Copied from transformers.models.bart.modeling_bart._expand_mask def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): """ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. @@ -126,9 +128,11 @@ def rotate_half(x): return torch.cat((-x2, x1), dim=-1) -def apply_rotary_pos_emb(q, k, cos, sin, offset: int = 0): - cos = cos[..., offset : q.shape[-2] + offset, :] - sin = sin[..., offset : q.shape[-2] + offset, :] +def apply_rotary_pos_emb(q, k, cos, sin, position_ids): + gather_indices = position_ids[:, None, :, None] # [bs, 1, seq_len, 1] + gather_indices = gather_indices.repeat(1, cos.shape[1], 1, cos.shape[3]) + cos = torch.gather(cos.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices) + sin = torch.gather(sin.repeat(gather_indices.shape[0], 1, 1, 1), 2, gather_indices) q_embed = (q * cos) + (rotate_half(q) * sin) k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed @@ -197,13 +201,12 @@ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): def forward( self, hidden_states: torch.Tensor, - past_key_value: Optional[Tuple[torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: bool = False, use_cache: bool = False, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """Input shape: Batch x Time x Channel""" - bsz, q_len, _ = hidden_states.size() query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) @@ -211,12 +214,10 @@ def forward( value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) kv_seq_len = key_states.shape[-2] - offset = 0 if past_key_value is not None: - offset = past_key_value[0].shape[-2] - kv_seq_len += offset + kv_seq_len += past_key_value[0].shape[-2] cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len) - query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, offset=offset) + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids) # [bsz, nh, t, hd] if past_key_value is not None: @@ -283,9 +284,10 @@ def forward( self, hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = False, - past_key_value: Optional[Tuple[torch.Tensor]] = None, ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: """ Args: @@ -308,8 +310,9 @@ def forward( # Self Attention hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, - past_key_value=past_key_value, attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, ) @@ -406,7 +409,11 @@ def _set_gradient_checkpointing(self, module, value=False): - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + [What are position IDs?](../glossary#position-ids) past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape @@ -488,10 +495,12 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em return combined_attention_mask + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) def forward( self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = None, @@ -499,49 +508,6 @@ def forward( output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BaseModelOutputWithPast]: - r""" - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -559,11 +525,23 @@ def forward( batch_size, seq_length, _ = inputs_embeds.shape else: raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + seq_length_with_past = seq_length past_key_values_length = 0 + if past_key_values is not None: past_key_values_length = past_key_values[0][0].shape[2] seq_length_with_past = seq_length_with_past + past_key_values_length + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) # embed positions @@ -608,12 +586,14 @@ def custom_forward(*inputs): create_custom_forward(decoder_layer), hidden_states, attention_mask, + position_ids, None, ) else: layer_outputs = decoder_layer( hidden_states, attention_mask=attention_mask, + position_ids=position_ids, past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, @@ -674,11 +654,13 @@ def set_decoder(self, decoder): def get_decoder(self): return self.model + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, @@ -689,52 +671,10 @@ def forward( ) -> Union[Tuple, CausalLMOutputWithPast]: r""" Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of - shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional - tensors are only required when the model is used as a decoder in a Sequence to Sequence model. - - Contains pre-computed hidden-states (key and values in the self-attention blocks and in the - cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all `decoder_input_ids` of shape `(batch_size, sequence_length)`. - inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - use_cache (`bool`, *optional*): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding - (see `past_key_values`). - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Returns: @@ -765,6 +705,7 @@ def forward( outputs = self.model( input_ids=input_ids, attention_mask=attention_mask, + position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, @@ -807,6 +748,14 @@ def prepare_inputs_for_generation( if past_key_values: input_ids = input_ids[:, -1:] + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step if inputs_embeds is not None and past_key_values is None: model_inputs = {"inputs_embeds": inputs_embeds} @@ -815,6 +764,7 @@ def prepare_inputs_for_generation( model_inputs.update( { + "position_ids": position_ids, "past_key_values": past_key_values, "use_cache": kwargs.get("use_cache"), "attention_mask": attention_mask, @@ -868,6 +818,7 @@ def forward( self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, @@ -886,8 +837,9 @@ def forward( transformer_outputs = self.model( input_ids, - past_key_values=past_key_values, attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, diff --git a/tests/models/gpt_neox/test_modeling_gpt_neox.py b/tests/models/gpt_neox/test_modeling_gpt_neox.py index 1798f01358ff..519b10a040ff 100644 --- a/tests/models/gpt_neox/test_modeling_gpt_neox.py +++ b/tests/models/gpt_neox/test_modeling_gpt_neox.py @@ -237,7 +237,7 @@ def test_feed_forward_chunking(self): @require_torch class GPTNeoXLanguageGenerationTest(unittest.TestCase): @slow - def test_lm_generate_codegen(self): + def test_lm_generate_gptneox(self): tokenizer = AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped") for checkpointing in [True, False]: model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped") From 87a1a3c4d0f3a5205caee983a163e35d589c6e47 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:55:55 +0200 Subject: [PATCH 201/277] [`bnb`] Force `requires_grad` to be `False` (#22396) for rg to be `False` --- src/transformers/utils/bitsandbytes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/utils/bitsandbytes.py b/src/transformers/utils/bitsandbytes.py index a49a620407fe..8dd7dd62d326 100644 --- a/src/transformers/utils/bitsandbytes.py +++ b/src/transformers/utils/bitsandbytes.py @@ -133,6 +133,8 @@ def replace_8bit_linear(model, threshold=6.0, modules_to_not_convert=None, curre has_fp16_weights=False, threshold=threshold, ) + # Force requires grad to False to avoid unexpected errors + model._modules[name].requires_grad_(False) # Remove the last key for recursion current_key_name.pop(-1) return model From 19eacee743eee1d9ca7f022dc494be84b49ac85e Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:12:42 -0400 Subject: [PATCH 202/277] Transformers env safetensors (#22400) * Report safetensors version in transformers-cli env * Styling * Trigger CI maybe --- src/transformers/commands/env.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/transformers/commands/env.py b/src/transformers/commands/env.py index fac0204f8cc8..aa0dccb579cc 100644 --- a/src/transformers/commands/env.py +++ b/src/transformers/commands/env.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib.util import platform from argparse import ArgumentParser import huggingface_hub from .. import __version__ as version -from ..utils import is_flax_available, is_tf_available, is_torch_available +from ..utils import is_flax_available, is_safetensors_available, is_tf_available, is_torch_available from . import BaseTransformersCLICommand @@ -33,6 +34,16 @@ def register_subcommand(parser: ArgumentParser): download_parser.set_defaults(func=info_command_factory) def run(self): + safetensors_version = "not installed" + if is_safetensors_available(): + import safetensors + + safetensors_version = safetensors.__version__ + elif importlib.util.find_spec("safetensors") is not None: + import safetensors + + safetensors_version = f"{safetensors.__version__} but is ignored because of PyTorch version too old." + pt_version = "not installed" pt_cuda_available = "NA" if is_torch_available(): @@ -73,6 +84,7 @@ def run(self): "Platform": platform.platform(), "Python version": platform.python_version(), "Huggingface_hub version": huggingface_hub.__version__, + "Safetensors version": f"{safetensors_version}", "PyTorch version (GPU?)": f"{pt_version} ({pt_cuda_available})", "Tensorflow version (GPU?)": f"{tf_version} ({tf_cuda_available})", "Flax version (CPU?/GPU?/TPU?)": f"{flax_version} ({jax_backend})", From e7e9246e608dc28fefa87c9a8b8b14890b329bcc Mon Sep 17 00:00:00 2001 From: NielsRogge <48327001+NielsRogge@users.noreply.github.com> Date: Mon, 27 Mar 2023 17:38:07 +0200 Subject: [PATCH 203/277] [Pix2Struct] Add support to resize embeddings (#22394) * First draft * Fix integration test * Remove script * Fix test and typos * Fix one more test * Skip tied embeddings test * Remove line * Address comments --- .../pix2struct/configuration_pix2struct.py | 42 +++---- .../models/pix2struct/modeling_pix2struct.py | 23 +++- .../pix2struct/test_modeling_pix2struct.py | 103 +++++++++++++++++- 3 files changed, 143 insertions(+), 25 deletions(-) diff --git a/src/transformers/models/pix2struct/configuration_pix2struct.py b/src/transformers/models/pix2struct/configuration_pix2struct.py index c8a866966cdf..8642602cf97d 100644 --- a/src/transformers/models/pix2struct/configuration_pix2struct.py +++ b/src/transformers/models/pix2struct/configuration_pix2struct.py @@ -35,17 +35,16 @@ class Pix2StructTextConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`Pix2StructTextModel`]. It is used to instantiate a Pix2Struct text model according to the specified arguments, defining the model architecture. Instantiating a - configuration with the defaults will yield a similar configuration to that of the `Pix2StructText` used by the - [base architectures](https://huggingface.co/google/pix2struct-textcaps-base). + configuration with the defaults will yield a similar configuration to that of the Pix2Struct text decoder used by + the [google/pix2struct-base](https://huggingface.co/google/pix2struct-base) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. - Args: vocab_size (`int`, *optional*, defaults to 50244): Vocabulary size of the `Pix2Struct` text model. Defines the number of different tokens that can be - represented by the `inputs_ids` passed when calling [`Pix2StructModel`]. + represented by the `inputs_ids` passed when calling [`Pix2StructTextModel`]. hidden_size (`int`, *optional*, defaults to 768): Dimensionality of the encoder layers and the pooler layer. d_kv (`int`, *optional*, defaults to 64): @@ -83,10 +82,10 @@ class Pix2StructTextConfig(PretrainedConfig): ```python >>> from transformers import Pix2StructTextConfig, Pix2StructTextModel - >>> # Initializing a Pix2StructTextConfig with Salesforce/pix2struct-vqa-base style configuration + >>> # Initializing a Pix2StructTextConfig with google/pix2struct-base style configuration >>> configuration = Pix2StructTextConfig() - >>> # Initializing a Pix2StructTextModel (with random weights) from the Salesforce/pix2struct-vqa-base style configuration + >>> # Initializing a Pix2StructTextModel (with random weights) from the google/pix2struct-base style configuration >>> model = Pix2StructTextModel(configuration) >>> # Accessing the model configuration @@ -118,6 +117,7 @@ def __init__( use_cache=False, pad_token_id=0, eos_token_id=1, + tie_word_embeddings=False, **kwargs, ): self.vocab_size = vocab_size @@ -143,6 +143,7 @@ def __init__( pad_token_id=pad_token_id, eos_token_id=eos_token_id, decoder_start_token_id=decoder_start_token_id, + tie_word_embeddings=tie_word_embeddings, **kwargs, ) @@ -168,14 +169,13 @@ def from_pretrained( class Pix2StructVisionConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`Pix2StructVisionModel`]. It is used to - instantiate a PIX2STRUCT vision model according to the specified arguments, defining the model architecture. + instantiate a Pix2Struct vision model according to the specified arguments, defining the model architecture. Instantiating a configuration defaults will yield a similar configuration to that of the Pix2Struct-base - [Salesforce/pix2struct-vqa-base](https://huggingface.co/Salesforce/pix2struct-vqa-base) architecture. + [google/pix2struct-base](https://huggingface.co/google/pix2struct-base) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. - Args: hidden_size (`int`, *optional*, defaults to 768): Dimensionality of the encoder layers and the pooler layer. @@ -223,10 +223,10 @@ class Pix2StructVisionConfig(PretrainedConfig): ```python >>> from transformers import Pix2StructVisionConfig, Pix2StructVisionModel - >>> # Initializing a Pix2StructVisionConfig with Salesforce/pix2struct-vqa-base style configuration + >>> # Initializing a Pix2StructVisionConfig with google/pix2struct-base style configuration >>> configuration = Pix2StructVisionConfig() - >>> # Initializing a Pix2StructVisionModel (with random weights) from the Salesforce/pix2struct-vqa-base style configuration + >>> # Initializing a Pix2StructVisionModel (with random weights) from the google/pix2struct-base style configuration >>> model = Pix2StructVisionModel(configuration) >>> # Accessing the model configuration @@ -301,11 +301,11 @@ def from_pretrained( class Pix2StructConfig(PretrainedConfig): r""" - [`Pix2StructConfig`] is the configuration class to store the configuration of a [`Pix2StructModel`]. It is used to - instantiate a PIX2STRUCT model according to the specified arguments, defining the text model and vision model - configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the - PIX2STRUCT-base [Salesforce/pix2struct-vqa-base](https://huggingface.co/Salesforce/pix2struct-vqa-base) - architecture. + [`Pix2StructConfig`] is the configuration class to store the configuration of a + [`Pix2StructForConditionalGeneration`]. It is used to instantiate a Pix2Struct model according to the specified + arguments, defining the text model and vision model configs. Instantiating a configuration with the defaults will + yield a similar configuration to that of the Pix2Struct-base + [google/pix2struct-base](https://huggingface.co/google/pix2struct-base) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. @@ -327,20 +327,20 @@ class Pix2StructConfig(PretrainedConfig): Example: ```python - >>> from transformers import Pix2StructConfig, Pix2StructModel + >>> from transformers import Pix2StructConfig, Pix2StructForConditionalGeneration - >>> # Initializing a Pix2StructConfig with Salesforce/pix2struct-vqa-base style configuration + >>> # Initializing a Pix2StructConfig with google/pix2struct-base style configuration >>> configuration = Pix2StructConfig() - >>> # Initializing a Pix2StructPModel (with random weights) from the Salesforce/pix2struct-vqa-base style configuration - >>> model = Pix2StructModel(configuration) + >>> # Initializing a Pix2StructForConditionalGeneration (with random weights) from the google/pix2struct-base style configuration + >>> model = Pix2StructForConditionalGeneration(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a Pix2StructConfig from a Pix2StructTextConfig and a Pix2StructVisionConfig - >>> # Initializing a PIX2STRUCTText and PIX2STRUCTVision configuration + >>> # Initializing a Pix2Struct text and Pix2Struct vision configuration >>> config_text = Pix2StructTextConfig() >>> config_vision = Pix2StructVisionConfig() diff --git a/src/transformers/models/pix2struct/modeling_pix2struct.py b/src/transformers/models/pix2struct/modeling_pix2struct.py index 1dcd22e0f44f..ead913e1df6e 100644 --- a/src/transformers/models/pix2struct/modeling_pix2struct.py +++ b/src/transformers/models/pix2struct/modeling_pix2struct.py @@ -1369,6 +1369,12 @@ def get_input_embeddings(self): def set_input_embeddings(self, new_embeddings): self.embed_tokens = new_embeddings + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + @add_start_docstrings_to_model_forward(PIX2STRUCT_TEXT_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) def forward( @@ -1626,12 +1632,25 @@ def __init__(self, config: Pix2StructConfig): self.post_init() def get_input_embeddings(self): - return self.shared + return self.decoder.get_input_embeddings() def set_input_embeddings(self, new_embeddings): - self.shared = new_embeddings self.decoder.set_input_embeddings(new_embeddings) + def get_output_embeddings(self) -> nn.Module: + return self.decoder.get_output_embeddings() + + def set_output_embeddings(self, new_embeddings): + self.decoder.set_output_embeddings(new_embeddings) + + def resize_token_embeddings(self, new_num_tokens: Optional[int] = None) -> nn.Embedding: + model_embeds = self.decoder.resize_token_embeddings(new_num_tokens) + + # update vocab size + self.config.text_config.vocab_size = new_num_tokens + + return model_embeds + def get_decoder(self): return self.decoder diff --git a/tests/models/pix2struct/test_modeling_pix2struct.py b/tests/models/pix2struct/test_modeling_pix2struct.py index 3896f8d84584..dc219bbd61ab 100644 --- a/tests/models/pix2struct/test_modeling_pix2struct.py +++ b/tests/models/pix2struct/test_modeling_pix2struct.py @@ -14,7 +14,7 @@ # limitations under the License. """ Testing suite for the PyTorch Pix2Struct model. """ - +import copy import inspect import os import tempfile @@ -396,7 +396,7 @@ class Pix2StructTextImageModelTest(ModelTesterMixin, unittest.TestCase): fx_compatible = False test_head_masking = False test_pruning = False - test_resize_embeddings = False + test_resize_embeddings = True test_attention_outputs = False test_torchscript = False @@ -526,6 +526,105 @@ def test_initialization(self): msg=f"Parameter {name} of model {model_class} seems not properly initialized", ) + # overwrite because `vocab_size` is not an attribute of `Pix2StructConfig` but rather `Pix2StructTextConfig` + def test_resize_tokens_embeddings(self): + original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if not self.test_resize_embeddings: + return + + for model_class in self.all_model_classes: + config = copy.deepcopy(original_config) + model = model_class(config) + model.to(torch_device) + + if self.model_tester.is_training is False: + model.eval() + + model_vocab_size = config.text_config.vocab_size + # Retrieve the embeddings and clone theme + model_embed = model.resize_token_embeddings(model_vocab_size) + cloned_embeddings = model_embed.weight.clone() + + # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size + model_embed = model.resize_token_embeddings(model_vocab_size + 10) + self.assertEqual(model.config.text_config.vocab_size, model_vocab_size + 10) + # Check that it actually resizes the embeddings matrix + self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10) + # Check that the model can still do a forward pass successfully (every parameter should be resized) + model(**self._prepare_for_class(inputs_dict, model_class)) + + # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size + model_embed = model.resize_token_embeddings(model_vocab_size - 15) + self.assertEqual(model.config.text_config.vocab_size, model_vocab_size - 15) + # Check that it actually resizes the embeddings matrix + self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15) + + # Check that the model can still do a forward pass successfully (every parameter should be resized) + # Decoder input ids should be clamped to the maximum size of the vocabulary + if "decoder_input_ids" in inputs_dict: + inputs_dict["decoder_input_ids"].clamp_(max=model_vocab_size - 15 - 1) + model(**self._prepare_for_class(inputs_dict, model_class)) + + # Check that adding and removing tokens has not modified the first part of the embedding matrix. + models_equal = True + for p1, p2 in zip(cloned_embeddings, model_embed.weight): + if p1.data.ne(p2.data).sum() > 0: + models_equal = False + + self.assertTrue(models_equal) + + # overwrite because `vocab_size` is not an attribute of `Pix2StructConfig` but rather `Pix2StructTextConfig` + def test_resize_embeddings_untied(self): + original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + if not self.test_resize_embeddings: + return + + original_config.tie_word_embeddings = False + + # if model cannot untied embeddings -> leave test + if original_config.tie_word_embeddings: + return + + for model_class in self.all_model_classes: + config = copy.deepcopy(original_config) + model = model_class(config).to(torch_device) + + # if no output embeddings -> leave test + if model.get_output_embeddings() is None: + continue + + # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size + model_vocab_size = config.text_config.vocab_size + model.resize_token_embeddings(model_vocab_size + 10) + self.assertEqual(model.config.text_config.vocab_size, model_vocab_size + 10) + output_embeds = model.get_output_embeddings() + self.assertEqual(output_embeds.weight.shape[0], model_vocab_size + 10) + # Check bias if present + if output_embeds.bias is not None: + self.assertEqual(output_embeds.bias.shape[0], model_vocab_size + 10) + # Check that the model can still do a forward pass successfully (every parameter should be resized) + model(**self._prepare_for_class(inputs_dict, model_class)) + + # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size + model.resize_token_embeddings(model_vocab_size - 15) + self.assertEqual(model.config.text_config.vocab_size, model_vocab_size - 15) + # Check that it actually resizes the embeddings matrix + output_embeds = model.get_output_embeddings() + self.assertEqual(output_embeds.weight.shape[0], model_vocab_size - 15) + # Check bias if present + if output_embeds.bias is not None: + self.assertEqual(output_embeds.bias.shape[0], model_vocab_size - 15) + # Check that the model can still do a forward pass successfully (every parameter should be resized) + # Decoder input ids should be clamped to the maximum size of the vocabulary + if "decoder_input_ids" in inputs_dict: + inputs_dict["decoder_input_ids"].clamp_(max=model_vocab_size - 15 - 1) + # Check that the model can still do a forward pass successfully (every parameter should be resized) + model(**self._prepare_for_class(inputs_dict, model_class)) + + @unittest.skip(reason="Pix2Struct doesn't use tied weights") + def test_tied_model_weights_key_ignore(self): + pass + def _create_and_check_torchscript(self, config, inputs_dict): if not self.test_torchscript: return From 10a7ea714aa28cc4ea749874bafbead5081134f0 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Mon, 27 Mar 2023 16:39:26 +0100 Subject: [PATCH 204/277] Trainer: move Seq2SeqTrainer imports under the typing guard (#22401) --- src/transformers/trainer_seq2seq.py | 33 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/transformers/trainer_seq2seq.py b/src/transformers/trainer_seq2seq.py index 0e277f3227e0..af072f056b52 100644 --- a/src/transformers/trainer_seq2seq.py +++ b/src/transformers/trainer_seq2seq.py @@ -14,39 +14,42 @@ from copy import deepcopy from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union import torch from torch import nn from torch.utils.data import Dataset -from .data.data_collator import DataCollator from .deepspeed import is_deepspeed_zero3_enabled from .generation.configuration_utils import GenerationConfig -from .modeling_utils import PreTrainedModel -from .tokenization_utils_base import PreTrainedTokenizerBase from .trainer import Trainer -from .trainer_callback import TrainerCallback -from .trainer_utils import EvalPrediction, PredictionOutput -from .training_args import TrainingArguments from .utils import logging +if TYPE_CHECKING: + from .data.data_collator import DataCollator + from .modeling_utils import PreTrainedModel + from .tokenization_utils_base import PreTrainedTokenizerBase + from .trainer_callback import TrainerCallback + from .trainer_utils import EvalPrediction, PredictionOutput + from .training_args import TrainingArguments + + logger = logging.get_logger(__name__) class Seq2SeqTrainer(Trainer): def __init__( self, - model: Union[PreTrainedModel, nn.Module] = None, - args: TrainingArguments = None, - data_collator: Optional[DataCollator] = None, + model: Union["PreTrainedModel", nn.Module] = None, + args: "TrainingArguments" = None, + data_collator: Optional["DataCollator"] = None, train_dataset: Optional[Dataset] = None, eval_dataset: Optional[Union[Dataset, Dict[str, Dataset]]] = None, - tokenizer: Optional[PreTrainedTokenizerBase] = None, - model_init: Optional[Callable[[], PreTrainedModel]] = None, - compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None, - callbacks: Optional[List[TrainerCallback]] = None, + tokenizer: Optional["PreTrainedTokenizerBase"] = None, + model_init: Optional[Callable[[], "PreTrainedModel"]] = None, + compute_metrics: Optional[Callable[["EvalPrediction"], Dict]] = None, + callbacks: Optional[List["TrainerCallback"]] = None, optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None), preprocess_logits_for_metrics: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None, ): @@ -161,7 +164,7 @@ def predict( ignore_keys: Optional[List[str]] = None, metric_key_prefix: str = "test", **gen_kwargs, - ) -> PredictionOutput: + ) -> "PredictionOutput": """ Run prediction and returns predictions and potential metrics. From 34b4d50f9596c2562f5c19243441f97890a78f2e Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Mon, 27 Mar 2023 18:04:28 +0100 Subject: [PATCH 205/277] Trainer: missing None check (#22404) missing None check --- src/transformers/trainer_seq2seq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/trainer_seq2seq.py b/src/transformers/trainer_seq2seq.py index af072f056b52..08280df104cb 100644 --- a/src/transformers/trainer_seq2seq.py +++ b/src/transformers/trainer_seq2seq.py @@ -281,7 +281,7 @@ def prediction_step( # in case the batch is shorter than max length, the output should be padded if generated_tokens.shape[-1] < gen_config.max_length: generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_config.max_length) - elif generated_tokens.shape[-1] < gen_config.max_new_tokens + 1: + elif gen_config.max_new_tokens is not None and generated_tokens.shape[-1] < gen_config.max_new_tokens + 1: generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_config.max_new_tokens + 1) with torch.no_grad(): From d144ad4456b7275e3282a19e6302c901e1cda954 Mon Sep 17 00:00:00 2001 From: Donny Greenberg Date: Mon, 27 Mar 2023 13:07:53 -0400 Subject: [PATCH 206/277] Hardware Auto-Setup for Examples (#22319) * Add initial remote hardware auto-setup docs * Fix a few typos and clarify some language * Add missing dependency * Update self-hosted launch script with Sylvain's comments. * Formatting. * Trigger CI * Style --- examples/README.md | 38 +++++++++++++++++++++ examples/run_on_remote.py | 69 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 examples/run_on_remote.py diff --git a/examples/README.md b/examples/README.md index 0a5ec752d392..c1cddd2e4734 100644 --- a/examples/README.md +++ b/examples/README.md @@ -94,3 +94,41 @@ Alternatively, you can switch your cloned 🤗 Transformers to a specific versio git checkout tags/v3.5.1 ``` and run the example command as usual afterward. + +## Running the Examples on Remote Hardware with Auto-Setup + +[run_on_remote.py](./run_on_remote.py) is a script that launches any example on remote self-hosted hardware, +with automatic hardware and environment setup. It uses [Runhouse](https://github.com/run-house/runhouse) to launch +on self-hosted hardware (e.g. in your own cloud account or on-premise cluster) but there are other options +for running remotely as well. You can easily customize the example used, command line arguments, dependencies, +and type of compute hardware, and then run the script to automatically launch the example. + +You can refer to +[hardware setup](https://runhouse-docs.readthedocs-hosted.com/en/main/rh_primitives/cluster.html#hardware-setup) +for more information about hardware and dependency setup with Runhouse, or this +[Colab tutorial](https://colab.research.google.com/drive/1sh_aNQzJX5BKAdNeXthTNGxKz7sM9VPc) for a more in-depth +walkthrough. + +You can run the script with the following commands: + +```bash +# First install runhouse: +pip install runhouse + +# For an on-demand V100 with whichever cloud provider you have configured: +python run_on_remote.py \ + --example pytorch/text-generation/run_generation.py \ + --model_type=gpt2 \ + --model_name_or_path=gpt2 \ + --prompt "I am a language model and" + +# For byo (bring your own) cluster: +python run_on_remote.py --host --user --key_path \ + --example + +# For on-demand instances +python run_on_remote.py --instance --provider \ + --example +``` + +You can also adapt the script to your own needs. \ No newline at end of file diff --git a/examples/run_on_remote.py b/examples/run_on_remote.py new file mode 100644 index 000000000000..cb499c540b66 --- /dev/null +++ b/examples/run_on_remote.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import shlex +import runhouse as rh + +if __name__ == "__main__": + # Refer to https://runhouse-docs.readthedocs-hosted.com/en/main/rh_primitives/cluster.html#hardware-setup for cloud access + # setup instructions, if using on-demand hardware + + # If user passes --user --host --key_path , fill them in as BYO cluster + # If user passes --instance --provider , fill them in as on-demand cluster + # Throw an error if user passes both BYO and on-demand cluster args + # Otherwise, use default values + parser = argparse.ArgumentParser() + parser.add_argument("--user", type=str, default="ubuntu") + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--key_path", type=str, default=None) + parser.add_argument("--instance", type=str, default="V100:1") + parser.add_argument("--provider", type=str, default="cheapest") + parser.add_argument("--use_spot", type=bool, default=False) + parser.add_argument("--example", type=str, default="pytorch/text-generation/run_generation.py") + args, unknown = parser.parse_known_args() + if args.host != "localhost": + if args.instance != "V100:1" or args.provider != "cheapest": + raise ValueError("Cannot specify both BYO and on-demand cluster args") + cluster = rh.cluster( + name="rh-cluster", ips=[args.host], ssh_creds={"ssh_user": args.user, "ssh_private_key": args.key_path} + ) + else: + cluster = rh.cluster( + name="rh-cluster", instance_type=args.instance, provider=args.provider, use_spot=args.use_spot + ) + example_dir = args.example.rsplit("/", 1)[0] + + # Set up remote environment + cluster.install_packages(["pip:./"]) # Installs transformers from local source + # Note transformers is copied into the home directory on the remote machine, so we can install from there + cluster.run([f"pip install -r transformers/examples/{example_dir}/requirements.txt"]) + cluster.run(["pip install torch --upgrade --extra-index-url https://download.pytorch.org/whl/cu117"]) + + # Run example. You can bypass the CLI wrapper and paste your own code here. + cluster.run([f'python transformers/examples/{args.example} {" ".join(shlex.quote(arg) for arg in unknown)}']) + + # Alternatively, we can just import and run a training function (especially if there's no wrapper CLI): + # from my_script... import train + # reqs = ['pip:./', 'torch', 'datasets', 'accelerate', 'evaluate', 'tqdm', 'scipy', 'scikit-learn', 'tensorboard'] + # launch_train_gpu = rh.function(fn=train, + # system=gpu, + # reqs=reqs, + # name='train_bert_glue') + # + # We can pass in arguments just like we would to a function: + # launch_train_gpu(num_epochs = 3, lr = 2e-5, seed = 42, batch_size = 16 + # stream_logs=True) From 2ec54c7a59cb31527e49d981ecb2e5d7f356b6e0 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger Date: Mon, 27 Mar 2023 13:17:14 -0400 Subject: [PATCH 207/277] Fix quality --- examples/run_on_remote.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/run_on_remote.py b/examples/run_on_remote.py index cb499c540b66..9d42ed845c9e 100644 --- a/examples/run_on_remote.py +++ b/examples/run_on_remote.py @@ -16,8 +16,10 @@ import argparse import shlex + import runhouse as rh + if __name__ == "__main__": # Refer to https://runhouse-docs.readthedocs-hosted.com/en/main/rh_primitives/cluster.html#hardware-setup for cloud access # setup instructions, if using on-demand hardware From 0cab4063861d44556328583a70e8af484418cfad Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Mon, 27 Mar 2023 19:42:00 +0200 Subject: [PATCH 208/277] [WIP]`NLLB-MoE` Adds the moe model (#22024) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial commit * update modeling code * update doc * add functions necessary * fix impotrs * revert changes * fixup * more styling to get going * remove standalone encoder * update code * styling * fix config and model * update code and some refactoring * make more tests pass * Adding NLLB-200 - MoE - 54.5B for no language left behind Fixes #21300 * fix mor common tests * styke * update testing file * update * update * Router2 doc * update check config with sparse layer * add dummy router * update current conversion script * create on the fly conversion script * Fixup * style * style 2 * fix empty return * fix return * Update default config sparse layers * easier to create sparse layers * update * update conversion script * update modeling * add to toctree * styling * make ruff happy * update docstring * update conversion script * update, will break tests but impelemting top2 * update * ❗local groups are supported here * ⚠️ Support for local groups is now removed ⚠️ This is because it has to work with model parallelism that we do not support * finish simplificaiton * Fix forward * style * fixup * Update modelling and test, refactoring * update tests * remove final layer)norm as it is done in the FF * routing works! Logits test added * nit in test * remove top1router * style * make sure sparse are tested. Had to change route_tokens a liottle bit * add support for unslip models when converting * fixup * style * update test s * update test * REFACTOR * encoder outputs match! * style * update testing * 🎉encoder and decoder logits match 🎉 * styleing * update tests * cleanup tests * fix router test and CIs * cleanup * cleanup test styling * fix tests * Finally the generation tests match! * cleanup * update test * style testing file * remove script * cleanup * more cleanup * nits * update * NLLB tokenizer is wrong and will be fixed soon * use LongTensors * update tests * revert some small changes * fix second expert sampling and batch prioritized routing * update tests * finish last tests * make ruff happy * update * ruff again * style * Update docs/source/en/model_doc/nllb-moe.mdx Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Updates based on review * style and fix import issue * nit * more nits * cleanup * styling * update test_seconde_expert_policy * fix name * last nit on the markdown examples --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- README.md | 1 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/en/_toctree.yml | 2 + docs/source/en/index.mdx | 2 + docs/source/en/model_doc/nllb-moe.mdx | 127 ++ docs/source/en/model_doc/nllb.mdx | 4 +- docs/source/en/tasks/summarization.mdx | 2 +- docs/source/en/tasks/translation.mdx | 2 +- src/transformers/__init__.py | 20 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + src/transformers/models/auto/modeling_auto.py | 4 + .../models/auto/tokenization_auto.py | 7 + src/transformers/models/nllb_moe/__init__.py | 68 + .../models/nllb_moe/configuration_nllb_moe.py | 218 ++ ..._sharded_original_checkpoint_to_pytorch.py | 160 ++ .../models/nllb_moe/modeling_nllb_moe.py | 1843 +++++++++++++++++ .../configuration_switch_transformers.py | 2 - src/transformers/utils/dummy_pt_objects.py | 38 + tests/models/nllb_moe/__init__.py | 0 .../models/nllb_moe/test_modeling_nllb_moe.py | 556 +++++ utils/check_repo.py | 2 + 27 files changed, 3063 insertions(+), 5 deletions(-) create mode 100644 docs/source/en/model_doc/nllb-moe.mdx create mode 100644 src/transformers/models/nllb_moe/__init__.py create mode 100644 src/transformers/models/nllb_moe/configuration_nllb_moe.py create mode 100644 src/transformers/models/nllb_moe/convert_nllb_moe_sharded_original_checkpoint_to_pytorch.py create mode 100644 src/transformers/models/nllb_moe/modeling_nllb_moe.py create mode 100644 tests/models/nllb_moe/__init__.py create mode 100644 tests/models/nllb_moe/test_modeling_nllb_moe.py diff --git a/README.md b/README.md index 2995b1b9c592..4dda4a2ae132 100644 --- a/README.md +++ b/README.md @@ -392,6 +392,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[NAT](https://huggingface.co/docs/transformers/model_doc/nat)** (from SHI Labs) released with the paper [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) by Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi. 1. **[Nezha](https://huggingface.co/docs/transformers/model_doc/nezha)** (from Huawei Noah’s Ark Lab) released with the paper [NEZHA: Neural Contextualized Representation for Chinese Language Understanding](https://arxiv.org/abs/1909.00204) by Junqiu Wei, Xiaozhe Ren, Xiaoguang Li, Wenyong Huang, Yi Liao, Yasheng Wang, Jiashu Lin, Xin Jiang, Xiao Chen and Qun Liu. 1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. +1. **[NLLB-MOE](https://huggingface.co/docs/transformers/main/model_doc/nllb-moe)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. 1. **[Nyströmformer](https://huggingface.co/docs/transformers/model_doc/nystromformer)** (from the University of Wisconsin - Madison) released with the paper [Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention](https://arxiv.org/abs/2102.03902) by Yunyang Xiong, Zhanpeng Zeng, Rudrasis Chakraborty, Mingxing Tan, Glenn Fung, Yin Li, Vikas Singh. 1. **[OneFormer](https://huggingface.co/docs/transformers/model_doc/oneformer)** (from SHI Labs) released with the paper [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) by Jitesh Jain, Jiachen Li, MangTik Chiu, Ali Hassani, Nikita Orlov, Humphrey Shi. 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. diff --git a/README_es.md b/README_es.md index c37dd68829da..5022db49326f 100644 --- a/README_es.md +++ b/README_es.md @@ -380,6 +380,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[NAT](https://huggingface.co/docs/transformers/model_doc/nat)** (from SHI Labs) released with the paper [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) by Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi. 1. **[Nezha](https://huggingface.co/docs/transformers/model_doc/nezha)** (from Huawei Noah’s Ark Lab) released with the paper [NEZHA: Neural Contextualized Representation for Chinese Language Understanding](https://arxiv.org/abs/1909.00204) by Junqiu Wei, Xiaozhe Ren, Xiaoguang Li, Wenyong Huang, Yi Liao, Yasheng Wang, Jiashu Lin, Xin Jiang, Xiao Chen and Qun Liu. 1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. +1. **[NLLB-MOE](https://huggingface.co/docs/transformers/main/model_doc/nllb-moe)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. 1. **[Nyströmformer](https://huggingface.co/docs/transformers/model_doc/nystromformer)** (from the University of Wisconsin - Madison) released with the paper [Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention](https://arxiv.org/abs/2102.03902) by Yunyang Xiong, Zhanpeng Zeng, Rudrasis Chakraborty, Mingxing Tan, Glenn Fung, Yin Li, Vikas Singh. 1. **[OneFormer](https://huggingface.co/docs/transformers/model_doc/oneformer)** (from SHI Labs) released with the paper [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) by Jitesh Jain, Jiachen Li, MangTik Chiu, Ali Hassani, Nikita Orlov, Humphrey Shi. 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. diff --git a/README_hd.md b/README_hd.md index 447e144b7b3b..a1cfe94d849f 100644 --- a/README_hd.md +++ b/README_hd.md @@ -352,6 +352,7 @@ conda install -c huggingface transformers 1. **[NAT](https://huggingface.co/docs/transformers/model_doc/nat)** (from SHI Labs) released with the paper [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) by Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi. 1. **[Nezha](https://huggingface.co/docs/transformers/model_doc/nezha)** (हुआवेई नूह के आर्क लैब से) साथ में कागज़ [NEZHA: चीनी भाषा समझ के लिए तंत्रिका प्रासंगिक प्रतिनिधित्व](https :/ /arxiv.org/abs/1909.00204) जुन्किउ वेई, ज़ियाओज़े रेन, ज़िआओगुआंग ली, वेनयोंग हुआंग, यी लियाओ, याशेंग वांग, जियाशू लिन, शिन जियांग, जिओ चेन और कुन लियू द्वारा। 1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (फ्रॉम मेटा) साथ में पेपर [नो लैंग्वेज लेफ्ट बिहाइंड: स्केलिंग ह्यूमन-सेंटेड मशीन ट्रांसलेशन] (https://arxiv.org/abs/2207.04672) एनएलएलबी टीम द्वारा प्रकाशित। +1. **[NLLB-MOE](https://huggingface.co/docs/transformers/main/model_doc/nllb-moe)** (Meta से) the NLLB team. द्वाराअनुसंधान पत्र [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) के साथ जारी किया गया 1. **[Nyströmformer](https://huggingface.co/docs/transformers/model_doc/nystromformer)** (विस्कॉन्सिन विश्वविद्यालय - मैडिसन से) साथ में कागज [Nyströmformer: A Nyström- आधारित एल्गोरिथम आत्म-ध्यान का अनुमान लगाने के लिए ](https://arxiv.org/abs/2102.03902) युनयांग ज़िओंग, झानपेंग ज़ेंग, रुद्रसिस चक्रवर्ती, मिंगक्सिंग टैन, ग्लेन फंग, यिन ली, विकास सिंह द्वारा पोस्ट किया गया। 1. **[OneFormer](https://huggingface.co/docs/transformers/model_doc/oneformer)** (SHI Labs से) पेपर [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) जितेश जैन, जिआचेन ली, मांगटिक चिउ, अली हसनी, निकिता ओरलोव, हम्फ्री शि के द्वारा जारी किया गया है। 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. diff --git a/README_ja.md b/README_ja.md index 5ae781aad4f1..f36113a634d7 100644 --- a/README_ja.md +++ b/README_ja.md @@ -414,6 +414,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[NAT](https://huggingface.co/docs/transformers/model_doc/nat)** (SHI Labs から) Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi から公開された研究論文: [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) 1. **[Nezha](https://huggingface.co/docs/transformers/model_doc/nezha)** (Huawei Noah’s Ark Lab から) Junqiu Wei, Xiaozhe Ren, Xiaoguang Li, Wenyong Huang, Yi Liao, Yasheng Wang, Jiashu Lin, Xin Jiang, Xiao Chen and Qun Liu から公開された研究論文: [NEZHA: Neural Contextualized Representation for Chinese Language Understanding](https://arxiv.org/abs/1909.00204) 1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (Meta から) the NLLB team から公開された研究論文: [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) +1. **[NLLB-MOE](https://huggingface.co/docs/transformers/main/model_doc/nllb-moe)** (Meta から) the NLLB team. から公開された研究論文 [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) 1. **[Nyströmformer](https://huggingface.co/docs/transformers/model_doc/nystromformer)** (the University of Wisconsin - Madison から) Yunyang Xiong, Zhanpeng Zeng, Rudrasis Chakraborty, Mingxing Tan, Glenn Fung, Yin Li, Vikas Singh から公開された研究論文: [Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention](https://arxiv.org/abs/2102.03902) 1. **[OneFormer](https://huggingface.co/docs/transformers/model_doc/oneformer)** (SHI Labs から) Jitesh Jain, Jiachen Li, MangTik Chiu, Ali Hassani, Nikita Orlov, Humphrey Shi から公開された研究論文: [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (Meta AI から) Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al から公開された研究論文: [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) diff --git a/README_ko.md b/README_ko.md index c87edecdb13d..9d26141de03a 100644 --- a/README_ko.md +++ b/README_ko.md @@ -329,6 +329,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[NAT](https://huggingface.co/docs/transformers/model_doc/nat)** (SHI Labs 에서) Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi 의 [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) 논문과 함께 발표했습니다. 1. **[Nezha](https://huggingface.co/docs/transformers/model_doc/nezha)** (Huawei Noah’s Ark Lab 에서) Junqiu Wei, Xiaozhe Ren, Xiaoguang Li, Wenyong Huang, Yi Liao, Yasheng Wang, Jiashu Lin, Xin Jiang, Xiao Chen and Qun Liu 의 [NEZHA: Neural Contextualized Representation for Chinese Language Understanding](https://arxiv.org/abs/1909.00204) 논문과 함께 발표했습니다. 1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (Meta 에서) the NLLB team 의 [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) 논문과 함께 발표했습니다. +1. **[NLLB-MOE](https://huggingface.co/docs/transformers/main/model_doc/nllb-moe)** (Meta 에서 제공)은 the NLLB team.의 [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672)논문과 함께 발표했습니다. 1. **[Nyströmformer](https://huggingface.co/docs/transformers/model_doc/nystromformer)** (the University of Wisconsin - Madison 에서) Yunyang Xiong, Zhanpeng Zeng, Rudrasis Chakraborty, Mingxing Tan, Glenn Fung, Yin Li, Vikas Singh 의 [Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention](https://arxiv.org/abs/2102.03902) 논문과 함께 발표했습니다. 1. **[OneFormer](https://huggingface.co/docs/transformers/model_doc/oneformer)** (SHI Labs 에서) Jitesh Jain, Jiachen Li, MangTik Chiu, Ali Hassani, Nikita Orlov, Humphrey Shi 의 [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) 논문과 함께 발표했습니다. 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (Meta AI 에서) Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al 의 [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) 논문과 함께 발표했습니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index b342dd4fae97..cf9d6f69d590 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -353,6 +353,7 @@ conda install -c huggingface transformers 1. **[NAT](https://huggingface.co/docs/transformers/model_doc/nat)** (来自 SHI Labs) 伴随论文 [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) 由 Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi 发布。 1. **[Nezha](https://huggingface.co/docs/transformers/model_doc/nezha)** (来自华为诺亚方舟实验室) 伴随论文 [NEZHA: Neural Contextualized Representation for Chinese Language Understanding](https://arxiv.org/abs/1909.00204) 由 Junqiu Wei, Xiaozhe Ren, Xiaoguang Li, Wenyong Huang, Yi Liao, Yasheng Wang, Jiashu Lin, Xin Jiang, Xiao Chen and Qun Liu 发布。 1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (来自 Meta) 伴随论文 [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) 由 the NLLB team 发布。 +1. **[NLLB-MOE](https://huggingface.co/docs/transformers/main/model_doc/nllb-moe)** (来自 Meta) 伴随论文 [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) 由 the NLLB team 发布。 1. **[Nyströmformer](https://huggingface.co/docs/transformers/model_doc/nystromformer)** (来自 the University of Wisconsin - Madison) 伴随论文 [Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention](https://arxiv.org/abs/2102.03902) 由 Yunyang Xiong, Zhanpeng Zeng, Rudrasis Chakraborty, Mingxing Tan, Glenn Fung, Yin Li, Vikas Singh 发布。 1. **[OneFormer](https://huggingface.co/docs/transformers/model_doc/oneformer)** (来自 SHI Labs) 伴随论文 [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) 由 Jitesh Jain, Jiachen Li, MangTik Chiu, Ali Hassani, Nikita Orlov, Humphrey Shi 发布。 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (来自 Meta AI) 伴随论文 [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) 由 Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index ea0f4e1047c5..d25cbb6057c5 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -365,6 +365,7 @@ conda install -c huggingface transformers 1. **[NAT](https://huggingface.co/docs/transformers/model_doc/nat)** (from SHI Labs) released with the paper [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) by Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi. 1. **[Nezha](https://huggingface.co/docs/transformers/model_doc/nezha)** (from Huawei Noah’s Ark Lab) released with the paper [NEZHA: Neural Contextualized Representation for Chinese Language Understanding](https://arxiv.org/abs/1909.00204) by Junqiu Wei, Xiaozhe Ren, Xiaoguang Li, Wenyong Huang, Yi Liao, Yasheng Wang, Jiashu Lin, Xin Jiang, Xiao Chen and Qun Liu. 1. **[NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. +1. **[NLLB-MOE](https://huggingface.co/docs/transformers/main/model_doc/nllb-moe)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. 1. **[Nyströmformer](https://huggingface.co/docs/transformers/model_doc/nystromformer)** (from the University of Wisconsin - Madison) released with the paper [Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention](https://arxiv.org/abs/2102.03902) by Yunyang Xiong, Zhanpeng Zeng, Rudrasis Chakraborty, Mingxing Tan, Glenn Fung, Yin Li, Vikas Singh. 1. **[OneFormer](https://huggingface.co/docs/transformers/model_doc/oneformer)** (from SHI Labs) released with the paper [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) by Jitesh Jain, Jiachen Li, MangTik Chiu, Ali Hassani, Nikita Orlov, Humphrey Shi. 1. **[OPT](https://huggingface.co/docs/transformers/master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 335d42d7b900..26dbe5767e56 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -357,6 +357,8 @@ title: NEZHA - local: model_doc/nllb title: NLLB + - local: model_doc/nllb-moe + title: NLLB-MoE - local: model_doc/nystromformer title: Nyströmformer - local: model_doc/opt diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 198f3d97de39..216425b29f62 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -166,6 +166,7 @@ The documentation is organized into five sections: 1. **[NAT](model_doc/nat)** (from SHI Labs) released with the paper [Neighborhood Attention Transformer](https://arxiv.org/abs/2204.07143) by Ali Hassani, Steven Walton, Jiachen Li, Shen Li, and Humphrey Shi. 1. **[Nezha](model_doc/nezha)** (from Huawei Noah’s Ark Lab) released with the paper [NEZHA: Neural Contextualized Representation for Chinese Language Understanding](https://arxiv.org/abs/1909.00204) by Junqiu Wei, Xiaozhe Ren, Xiaoguang Li, Wenyong Huang, Yi Liao, Yasheng Wang, Jiashu Lin, Xin Jiang, Xiao Chen and Qun Liu. 1. **[NLLB](model_doc/nllb)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. +1. **[NLLB-MOE](model_doc/nllb-moe)** (from Meta) released with the paper [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by the NLLB team. 1. **[Nyströmformer](model_doc/nystromformer)** (from the University of Wisconsin - Madison) released with the paper [Nyströmformer: A Nyström-Based Algorithm for Approximating Self-Attention](https://arxiv.org/abs/2102.03902) by Yunyang Xiong, Zhanpeng Zeng, Rudrasis Chakraborty, Mingxing Tan, Glenn Fung, Yin Li, Vikas Singh. 1. **[OneFormer](model_doc/oneformer)** (from SHI Labs) released with the paper [OneFormer: One Transformer to Rule Universal Image Segmentation](https://arxiv.org/abs/2211.06220) by Jitesh Jain, Jiachen Li, MangTik Chiu, Ali Hassani, Nikita Orlov, Humphrey Shi. 1. **[OPT](master/model_doc/opt)** (from Meta AI) released with the paper [OPT: Open Pre-trained Transformer Language Models](https://arxiv.org/abs/2205.01068) by Susan Zhang, Stephen Roller, Naman Goyal, Mikel Artetxe, Moya Chen, Shuohui Chen et al. @@ -358,6 +359,7 @@ Flax), PyTorch, and/or TensorFlow. | MVP | ✅ | ✅ | ✅ | ❌ | ❌ | | NAT | ❌ | ❌ | ✅ | ❌ | ❌ | | Nezha | ❌ | ❌ | ✅ | ❌ | ❌ | +| NLLB-MOE | ❌ | ❌ | ✅ | ❌ | ❌ | | Nyströmformer | ❌ | ❌ | ✅ | ❌ | ❌ | | OneFormer | ❌ | ❌ | ✅ | ❌ | ❌ | | OpenAI GPT | ✅ | ✅ | ✅ | ✅ | ❌ | diff --git a/docs/source/en/model_doc/nllb-moe.mdx b/docs/source/en/model_doc/nllb-moe.mdx new file mode 100644 index 000000000000..75b0a9f580c3 --- /dev/null +++ b/docs/source/en/model_doc/nllb-moe.mdx @@ -0,0 +1,127 @@ + + +# NLLB-MOE + + +## Overview + +The NLLB model was presented in [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) by Marta R. Costa-jussà, James Cross, Onur Çelebi, +Maha Elbayad, Kenneth Heafield, Kevin Heffernan, Elahe Kalbassi, Janice Lam, Daniel Licht, Jean Maillard, Anna Sun, Skyler Wang, Guillaume Wenzek, Al Youngblood, Bapi Akula, +Loic Barrault, Gabriel Mejia Gonzalez, Prangthip Hansanti, John Hoffman, Semarley Jarrett, Kaushik Ram Sadagopan, Dirk Rowe, Shannon Spruit, Chau Tran, Pierre Andrews, +Necip Fazil Ayan, Shruti Bhosale, Sergey Edunov, Angela Fan, Cynthia Gao, Vedanuj Goswami, Francisco Guzmán, Philipp Koehn, Alexandre Mourachko, Christophe Ropers, +Safiyyah Saleem, Holger Schwenk, and Jeff Wang. + +The abstract of the paper is the following: + +*Driven by the goal of eradicating language barriers on a global scale, machine translation has solidified itself as a key focus of artificial intelligence research today. +However, such efforts have coalesced around a small subset of languages, leaving behind the vast majority of mostly low-resource languages. What does it take to break the +200 language barrier while ensuring safe, high quality results, all while keeping ethical considerations in mind? In No Language Left Behind, we took on this challenge by +first contextualizing the need for low-resource language translation support through exploratory interviews with native speakers. Then, we created datasets and models aimed +at narrowing the performance gap between low and high-resource languages. More specifically, we developed a conditional compute model based on Sparsely Gated Mixture of +Experts that is trained on data obtained with novel and effective data mining techniques tailored for low-resource languages. We propose multiple architectural and training +improvements to counteract overfitting while training on thousands of tasks. Critically, we evaluated the performance of over 40,000 different translation directions using +a human-translated benchmark, Flores-200, and combined human evaluation with a novel toxicity benchmark covering all languages in Flores-200 to assess translation safety. +Our model achieves an improvement of 44% BLEU relative to the previous state-of-the-art, laying important groundwork towards realizing a universal translation system.* + +Tips: + +- M2M100ForConditionalGeneration is the base model for both NLLB and NLLB MoE +- The NLLB-MoE is very similar to the NLLB model, but it's feed forward layer is based on the implementation of SwitchTransformers. +- The tokenizer is the same as the NLLB models. + +This model was contributed by [Arthur Zucker](https://huggingface.co/ArtZucker). +The original code can be found [here](https://github.com/facebookresearch/fairseq). + +## Implementation differences with SwitchTransformers +The biggest difference is the way the tokens are routed. NLLB-MoE uses a `top-2-gate` which means that blah blah blah blah. +In SwitchTransformers, once the masks are computed for each experts, we just index the current hidden_states with the routing mask, and feed the +correct tokens to the expert. However here, the implementation varies a lot as the fairseq repository used a different approach. + +## Generating with NLLB-MoE +The avalable checkpoints requires around 350GB of storage. Make sure to use `accelerate` if you do not have enough RAM on your machine. + +While generating the target text set the `forced_bos_token_id` to the target language id. The following +example shows how to translate English to French using the *facebook/nllb-200-distilled-600M* model. + +Note that we're using the BCP-47 code for French `fra_Latn`. See [here](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200) +for the list of all BCP-47 in the Flores 200 dataset. + +```python +>>> from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + +>>> tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-moe-54b") +>>> model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-moe-54b") + +>>> article = "Previously, Ring's CEO, Jamie Siminoff, remarked the company started when his doorbell wasn't audible from his shop in his garage." +>>> inputs = tokenizer(article, return_tensors="pt") + +>>> translated_tokens = model.generate( +... **inputs, forced_bos_token_id=tokenizer.lang_code_to_id["fra_Latn"], max_length=50 +... ) +>>> tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0] +"Auparavant, le PDG de Ring, Jamie Siminoff, a fait remarquer que la société avait commencé lorsque sa sonnette n'était pas audible depuis son magasin dans son garage." +``` + +### Generating from any other language than English + +English (`eng_Latn`) is set as the default language from which to translate. In order to specify that you'd like to translate from a different language, +you should specify the BCP-47 code in the `src_lang` keyword argument of the tokenizer initialization. + +See example below for a translation from romanian to german: + +```python +>>> from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + +>>> tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-moe-54b", src_lang="ron_Latn") +>>> model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-moe-54b") + +>>> article = "Şeful ONU spune că nu există o soluţie militară în Siria" +>>> inputs = tokenizer(article, return_tensors="pt") + +>>> translated_tokens = model.generate( +... **inputs, forced_bos_token_id=tokenizer.lang_code_to_id["deu_Latn"], max_length=30 +... ) +>>> tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0] +``` + +## Documentation resources + +- [Translation task guide](./tasks/translation) +- [Summarization task guide](./tasks/summarization) + + +## NllbMoeConfig + +[[autodoc]] NllbMoeConfig + +## NllbMoeTop2Router + +[[autodoc]] NllbMoeTop2Router + - route_tokens + - forward + +## NllbMoeSparseMLP + +[[autodoc]] NllbMoeSparseMLP + - forward + +## NllbMoeModel + +[[autodoc]] NllbMoeModel + - forward + +## NllbMoeForConditionalGeneration + +[[autodoc]] NllbMoeForConditionalGeneration + - forward + diff --git a/docs/source/en/model_doc/nllb.mdx b/docs/source/en/model_doc/nllb.mdx index a53f718894af..9ad286164ba0 100644 --- a/docs/source/en/model_doc/nllb.mdx +++ b/docs/source/en/model_doc/nllb.mdx @@ -35,7 +35,9 @@ improvements to counteract overfitting while training on thousands of tasks. Cri a human-translated benchmark, Flores-200, and combined human evaluation with a novel toxicity benchmark covering all languages in Flores-200 to assess translation safety. Our model achieves an improvement of 44% BLEU relative to the previous state-of-the-art, laying important groundwork towards realizing a universal translation system.* -This implementation contains the dense models available on release. Let us know via a GitHub issue if you would like to see the MoE models as well. +This implementation contains the dense models available on release. + +**The sparse model NLLB-MoE (Mixture of Expert) is now available! More details [here](nllb-moe)** This model was contributed by [Lysandre](https://huggingface.co/lysandre). The authors' code can be found [here](https://github.com/facebookresearch/fairseq/tree/nllb). diff --git a/docs/source/en/tasks/summarization.mdx b/docs/source/en/tasks/summarization.mdx index efb2db60bda3..a9b890dcea97 100644 --- a/docs/source/en/tasks/summarization.mdx +++ b/docs/source/en/tasks/summarization.mdx @@ -31,7 +31,7 @@ The task illustrated in this tutorial is supported by the following model archit -[BART](../model_doc/bart), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [Encoder decoder](../model_doc/encoder-decoder), [FairSeq Machine-Translation](../model_doc/fsmt), [GPTSAN-japanese](../model_doc/gptsan-japanese), [LED](../model_doc/led), [LongT5](../model_doc/longt5), [M2M100](../model_doc/m2m_100), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [MT5](../model_doc/mt5), [MVP](../model_doc/mvp), [NLLB](../model_doc/nllb), [Pegasus](../model_doc/pegasus), [PEGASUS-X](../model_doc/pegasus_x), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [SwitchTransformers](../model_doc/switch_transformers), [T5](../model_doc/t5), [XLM-ProphetNet](../model_doc/xlm-prophetnet) +[BART](../model_doc/bart), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [Encoder decoder](../model_doc/encoder-decoder), [FairSeq Machine-Translation](../model_doc/fsmt), [GPTSAN-japanese](../model_doc/gptsan-japanese), [LED](../model_doc/led), [LongT5](../model_doc/longt5), [M2M100](../model_doc/m2m_100), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [MT5](../model_doc/mt5), [MVP](../model_doc/mvp), [NLLB](../model_doc/nllb), [NLLB-MOE](../model_doc/nllb-moe), [Pegasus](../model_doc/pegasus), [PEGASUS-X](../model_doc/pegasus_x), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [SwitchTransformers](../model_doc/switch_transformers), [T5](../model_doc/t5), [XLM-ProphetNet](../model_doc/xlm-prophetnet) diff --git a/docs/source/en/tasks/translation.mdx b/docs/source/en/tasks/translation.mdx index df9df832daf9..361efbf54cc5 100644 --- a/docs/source/en/tasks/translation.mdx +++ b/docs/source/en/tasks/translation.mdx @@ -28,7 +28,7 @@ The task illustrated in this tutorial is supported by the following model archit -[BART](../model_doc/bart), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [Encoder decoder](../model_doc/encoder-decoder), [FairSeq Machine-Translation](../model_doc/fsmt), [GPTSAN-japanese](../model_doc/gptsan-japanese), [LED](../model_doc/led), [LongT5](../model_doc/longt5), [M2M100](../model_doc/m2m_100), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [MT5](../model_doc/mt5), [MVP](../model_doc/mvp), [NLLB](../model_doc/nllb), [Pegasus](../model_doc/pegasus), [PEGASUS-X](../model_doc/pegasus_x), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [SwitchTransformers](../model_doc/switch_transformers), [T5](../model_doc/t5), [XLM-ProphetNet](../model_doc/xlm-prophetnet) +[BART](../model_doc/bart), [BigBird-Pegasus](../model_doc/bigbird_pegasus), [Blenderbot](../model_doc/blenderbot), [BlenderbotSmall](../model_doc/blenderbot-small), [Encoder decoder](../model_doc/encoder-decoder), [FairSeq Machine-Translation](../model_doc/fsmt), [GPTSAN-japanese](../model_doc/gptsan-japanese), [LED](../model_doc/led), [LongT5](../model_doc/longt5), [M2M100](../model_doc/m2m_100), [Marian](../model_doc/marian), [mBART](../model_doc/mbart), [MT5](../model_doc/mt5), [MVP](../model_doc/mvp), [NLLB](../model_doc/nllb), [NLLB-MOE](../model_doc/nllb-moe), [Pegasus](../model_doc/pegasus), [PEGASUS-X](../model_doc/pegasus_x), [PLBart](../model_doc/plbart), [ProphetNet](../model_doc/prophetnet), [SwitchTransformers](../model_doc/switch_transformers), [T5](../model_doc/t5), [XLM-ProphetNet](../model_doc/xlm-prophetnet) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index cffd17d428fc..52a770bc5c7e 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -386,6 +386,7 @@ "models.nat": ["NAT_PRETRAINED_CONFIG_ARCHIVE_MAP", "NatConfig"], "models.nezha": ["NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP", "NezhaConfig"], "models.nllb": [], + "models.nllb_moe": ["NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP", "NllbMoeConfig"], "models.nystromformer": [ "NYSTROMFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP", "NystromformerConfig", @@ -2045,6 +2046,16 @@ "NezhaPreTrainedModel", ] ) + _import_structure["models.nllb_moe"].extend( + [ + "NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST", + "NllbMoeForConditionalGeneration", + "NllbMoeModel", + "NllbMoePreTrainedModel", + "NllbMoeSparseMLP", + "NllbMoeTop2Router", + ] + ) _import_structure["models.nystromformer"].extend( [ "NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -4035,6 +4046,7 @@ from .models.mvp import MvpConfig, MvpTokenizer from .models.nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig from .models.nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig + from .models.nllb_moe import NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP, NllbMoeConfig from .models.nystromformer import NYSTROMFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, NystromformerConfig from .models.oneformer import ONEFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, OneFormerConfig, OneFormerProcessor from .models.openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, OpenAIGPTTokenizer @@ -5423,6 +5435,14 @@ NezhaModel, NezhaPreTrainedModel, ) + from .models.nllb_moe import ( + NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST, + NllbMoeForConditionalGeneration, + NllbMoeModel, + NllbMoePreTrainedModel, + NllbMoeSparseMLP, + NllbMoeTop2Router, + ) from .models.nystromformer import ( NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST, NystromformerForMaskedLM, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index e6e936148b61..a086b879650a 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -132,6 +132,7 @@ nat, nezha, nllb, + nllb_moe, nystromformer, oneformer, openai, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 32c5da7bd143..03492c0c5aad 100755 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -134,6 +134,7 @@ ("mvp", "MvpConfig"), ("nat", "NatConfig"), ("nezha", "NezhaConfig"), + ("nllb-moe", "NllbMoeConfig"), ("nystromformer", "NystromformerConfig"), ("oneformer", "OneFormerConfig"), ("openai-gpt", "OpenAIGPTConfig"), @@ -310,6 +311,7 @@ ("mvp", "MVP_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("nat", "NAT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("nezha", "NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("nllb-moe", "NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("nystromformer", "NYSTROMFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("oneformer", "ONEFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("openai-gpt", "OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -501,6 +503,7 @@ ("nat", "NAT"), ("nezha", "Nezha"), ("nllb", "NLLB"), + ("nllb-moe", "NLLB-MOE"), ("nystromformer", "Nyströmformer"), ("oneformer", "OneFormer"), ("openai-gpt", "OpenAI GPT"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index e8f295d1c12d..f24146199876 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -132,6 +132,7 @@ ("mvp", "MvpModel"), ("nat", "NatModel"), ("nezha", "NezhaModel"), + ("nllb-moe", "NllbMoeModel"), ("nystromformer", "NystromformerModel"), ("oneformer", "OneFormerModel"), ("openai-gpt", "OpenAIGPTModel"), @@ -235,6 +236,7 @@ ("mpnet", "MPNetForMaskedLM"), ("mvp", "MvpForConditionalGeneration"), ("nezha", "NezhaForPreTraining"), + ("nllb-moe", "NllbMoeForConditionalGeneration"), ("openai-gpt", "OpenAIGPTLMHeadModel"), ("retribert", "RetriBertModel"), ("roberta", "RobertaForMaskedLM"), @@ -310,6 +312,7 @@ ("mpnet", "MPNetForMaskedLM"), ("mvp", "MvpForConditionalGeneration"), ("nezha", "NezhaForMaskedLM"), + ("nllb-moe", "NllbMoeForConditionalGeneration"), ("nystromformer", "NystromformerForMaskedLM"), ("openai-gpt", "OpenAIGPTLMHeadModel"), ("pegasus_x", "PegasusXForConditionalGeneration"), @@ -604,6 +607,7 @@ ("mbart", "MBartForConditionalGeneration"), ("mt5", "MT5ForConditionalGeneration"), ("mvp", "MvpForConditionalGeneration"), + ("nllb-moe", "NllbMoeForConditionalGeneration"), ("pegasus", "PegasusForConditionalGeneration"), ("pegasus_x", "PegasusXForConditionalGeneration"), ("plbart", "PLBartForConditionalGeneration"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 900c0b836d8e..fd10c5cd1176 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -216,6 +216,13 @@ "NllbTokenizerFast" if is_tokenizers_available() else None, ), ), + ( + "nllb-moe", + ( + "NllbTokenizer" if is_sentencepiece_available() else None, + "NllbTokenizerFast" if is_tokenizers_available() else None, + ), + ), ( "nystromformer", ( diff --git a/src/transformers/models/nllb_moe/__init__.py b/src/transformers/models/nllb_moe/__init__.py new file mode 100644 index 000000000000..ea0f7752ed0c --- /dev/null +++ b/src/transformers/models/nllb_moe/__init__.py @@ -0,0 +1,68 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available + + +_import_structure = { + "configuration_nllb_moe": [ + "NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP", + "NllbMoeConfig", + ] +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_nllb_moe"] = [ + "NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST", + "NllbMoeForConditionalGeneration", + "NllbMoeModel", + "NllbMoePreTrainedModel", + "NllbMoeTop2Router", + "NllbMoeSparseMLP", + ] + + +if TYPE_CHECKING: + from .configuration_nllb_moe import ( + NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP, + NllbMoeConfig, + ) + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_nllb_moe import ( + NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST, + NllbMoeForConditionalGeneration, + NllbMoeModel, + NllbMoePreTrainedModel, + NllbMoeSparseMLP, + NllbMoeTop2Router, + ) + + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/nllb_moe/configuration_nllb_moe.py b/src/transformers/models/nllb_moe/configuration_nllb_moe.py new file mode 100644 index 000000000000..03a37bb35d6b --- /dev/null +++ b/src/transformers/models/nllb_moe/configuration_nllb_moe.py @@ -0,0 +1,218 @@ +# coding=utf-8 +# Copyright 2023, HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" NLLB-MoE model configuration""" +from ...configuration_utils import PretrainedConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "facebook/nllb-moe-54B": "https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json", +} + + +class NllbMoeConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`NllbMoeModel`]. It is used to instantiate an + NLLB-MoE model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the NLLB-MoE + [facebook/nllb-moe-54b](https://huggingface.co/facebook/nllb-moe-54b) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 50265): + Vocabulary size of the NllbMoe model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`NllbMoeModel`] or + d_model (`int`, *optional*, defaults to 1024): + Dimensionality of the layers and the pooler layer. + encoder_layers (`int`, *optional*, defaults to 12): + Number of encoder layers. + decoder_layers (`int`, *optional*, defaults to 12): + Number of decoder layers. + encoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + decoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer decoder. + decoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. + encoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. + activation_function (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + dropout (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + activation_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + classifier_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for classifier. + max_position_embeddings (`int`, *optional*, defaults to 1024): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + init_std (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + encoder_layerdrop (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + decoder_layerdrop (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + second_expert_policy ( `str`, *optional*, default to `"all"`): + The policy used for the sampling the probability of being sampled to a second expert for each token. + normalize_router_prob_before_dropping (`bool`, *optional*, defaults to `True`): + Whether or not to normalize the router probabilities before applying a mask based on the experts capacity + (capacity dropping). + batch_prioritized_routing (`bool`, *optional*, defaults to `True`): + Whether or not to orders the tokens by their router probabilities before capacity dropping. This means that + the tokens that have the highest probabilities will be routed before other tokens that might be further in + the sequence. + moe_eval_capacity_token_fraction (`float`, *optional*, defaults to 1.0): + Fraction of tokens as capacity during validation, if set to negative, uses the same as training. Should be + in range: (0.0, 1.0]. + num_experts (`int`, *optional*, defaults to 128): + Number of experts for each NllbMoeSparseMlp layer. + expert_capacity (`int`, *optional*, defaults to 64): + Number of tokens that can be stored in each expert. + encoder_sparse_step (`int`, *optional*, defaults to 4): + Frequency of the sparse layers in the encoder. 4 means that one out of 4 layers will be sparse. + decoder_sparse_step (`int`, *optional*, defaults to 4): + Frequency of the sparse layers in the decoder. 4 means that one out of 4 layers will be sparse. + router_dtype (`str`, *optional*, default to `"float32"`): + The `dtype` used for the routers. It is preferable to keep the `dtype` to `"float32"` as specified in the + *selective precision* discussion in [the paper](https://arxiv.org/abs/2101.03961). + router_ignore_padding_tokens (`bool`, *optional*, defaults to `False`): + Whether to ignore padding tokens when routing. if `False`, the padding tokens are not routed to any + experts. + router_bias (`bool`, *optional*, defaults to `False`): + Whether or not the classifier of the router should have a bias. + moe_token_dropout (`float`, *optional*, defualt ot 0.2): + Masking rate for MoE expert output masking (EOM), which is implemented via a Dropout2d on the expert + outputs. + output_router_logits (`bool`, *optional*, defaults to `False`): + Whether or not to return the router logits. Only set to `True` to get the auxiliary loss when training. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + + Example: + + ```python + >>> from transformers import NllbMoeModel, NllbMoeConfig + + >>> # Initializing a NllbMoe facebook/nllb-moe-54b style configuration + >>> configuration = NllbMoeConfig() + + >>> # Initializing a model from the facebook/nllb-moe-54b style configuration + >>> model = NllbMoeModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "nllb_moe" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"} + + def __init__( + self, + vocab_size=128112, + max_position_embeddings=1024, + encoder_layers=12, + encoder_ffn_dim=4096, + encoder_attention_heads=16, + decoder_layers=12, + decoder_ffn_dim=4096, + decoder_attention_heads=16, + encoder_layerdrop=0.05, + decoder_layerdrop=0.05, + use_cache=True, + is_encoder_decoder=True, + activation_function="relu", + d_model=1024, + dropout=0.1, + attention_dropout=0.1, + activation_dropout=0.0, + init_std=0.02, + decoder_start_token_id=2, + scale_embedding=True, + router_bias=False, + router_dtype="float32", + router_ignore_padding_tokens=False, + num_experts=128, + expert_capacity=64, + encoder_sparse_step=4, + decoder_sparse_step=4, + router_z_loss_coef=0.001, + router_aux_loss_coef=0.001, + second_expert_policy="all", + normalize_router_prob_before_dropping=False, + batch_prioritized_routing=False, + moe_eval_capacity_token_fraction=1.0, + moe_token_dropout=0.2, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + output_router_logits=False, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.d_model = d_model + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_layers = encoder_layers + self.encoder_attention_heads = encoder_attention_heads + self.decoder_ffn_dim = decoder_ffn_dim + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation_dropout = activation_dropout + self.activation_function = activation_function + self.init_std = init_std + self.encoder_layerdrop = encoder_layerdrop + self.decoder_layerdrop = decoder_layerdrop + self.use_cache = use_cache + self.num_hidden_layers = encoder_layers + self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + self.router_z_loss_coef = router_z_loss_coef + self.router_aux_loss_coef = router_aux_loss_coef + self.decoder_sparse_step = decoder_sparse_step + self.encoder_sparse_step = encoder_sparse_step + self.num_experts = num_experts + self.expert_capacity = expert_capacity + self.router_bias = router_bias + if router_dtype not in ["float32", "float16", "bfloat16"]: + raise ValueError(f"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}") + self.router_dtype = router_dtype + + self.router_ignore_padding_tokens = router_ignore_padding_tokens + self.batch_prioritized_routing = batch_prioritized_routing + self.second_expert_policy = second_expert_policy + self.normalize_router_prob_before_dropping = normalize_router_prob_before_dropping + self.moe_eval_capacity_token_fraction = moe_eval_capacity_token_fraction + self.moe_token_dropout = moe_token_dropout + self.output_router_logits = output_router_logits + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + is_encoder_decoder=is_encoder_decoder, + decoder_start_token_id=decoder_start_token_id, + **kwargs, + ) diff --git a/src/transformers/models/nllb_moe/convert_nllb_moe_sharded_original_checkpoint_to_pytorch.py b/src/transformers/models/nllb_moe/convert_nllb_moe_sharded_original_checkpoint_to_pytorch.py new file mode 100644 index 000000000000..5f98c0ca3d92 --- /dev/null +++ b/src/transformers/models/nllb_moe/convert_nllb_moe_sharded_original_checkpoint_to_pytorch.py @@ -0,0 +1,160 @@ +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os + +import torch +from torch import nn + +from transformers import NllbMoeConfig, NllbMoeModel +from transformers.modeling_utils import dtype_byte_size +from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME + + +def remove_ignore_keys_(state_dict): + ignore_keys = [ + "encoder.version", + "decoder.version", + "model.encoder.version", + "model.decoder.version", + "decoder.output_projection.weight", + "_float_tensor", + "encoder.embed_positions._float_tensor", + "decoder.embed_positions._float_tensor", + ] + for k in ignore_keys: + state_dict.pop(k, None) + + +def make_linear_from_emb(emb): + vocab_size, emb_size = emb.weight.shape + lin_layer = nn.Linear(vocab_size, emb_size, bias=False) + lin_layer.weight.data = emb.weight.data + return lin_layer + + +def rename_fairseq_keys(state_dict, expert_idx=None): + new_dict = {} + for old_key in state_dict.keys(): + key = old_key + if "moe_layer.experts." in key: + if expert_idx is not None: + key = key.replace("moe_layer.experts.0", f"ffn.experts.expert_{expert_idx}") + else: + key = key.replace("moe_layer.experts.", "ffn.experts.expert_") + if "gate" in key: + key = key.replace(".moe_layer.gate.wg", ".ffn.router.classifier") + if "fc2" and "experts" not in key: + key = key.replace(".fc2.", ".ffn.fc2.") + if "fc1" and "experts" not in key: + key = key.replace(".fc1.", ".ffn.fc1.") + if ".encoder_attn." in key: + key = key.replace(".encoder_attn.", ".cross_attention.") + if "encoder_attn_layer_norm" in key: + key = key.replace("encoder_attn_layer_norm", "cross_attention_layer_norm") + if "final_layer_norm" in key: + key = key.replace("final_layer_norm", "ff_layer_norm") + new_dict[key] = state_dict[old_key] + return new_dict + + +def shard_on_the_fly(switch_checkpoint_path, dump_path, num_experts, dtype, weights_name: str = WEIGHTS_NAME): + sharded_state_dicts = [] + total_size = 0 + os.makedirs(dump_path, exist_ok=True) + + for expert in range(num_experts): + expert_path = switch_checkpoint_path + f"-rank-{expert}.pt" + if os.path.isfile(expert_path): + expert_state = torch.load(expert_path)["model"] + remove_ignore_keys_(expert_state) + expert_state = rename_fairseq_keys(expert_state, expert) + save_path = os.path.join( + dump_path, weights_name.replace(".bin", f"-{len(sharded_state_dicts)+1:05d}-of-???.bin") + ) + torch.save(expert_state, save_path) + sharded_state_dicts.append(expert_state.keys()) + total_size += sum([value.numel() for key, value in expert_state.items()]) * dtype_byte_size( + expert_state[list(expert_state)[0]].dtype + ) + + # Add the last block + save_path = os.path.join(dump_path, weights_name.replace(".bin", f"-{len(sharded_state_dicts)+1:05d}-of-???.bin")) + shared_weights = torch.load(switch_checkpoint_path + "-shared.pt")["model"] + remove_ignore_keys_(shared_weights) + shared_weights = rename_fairseq_keys(shared_weights, None) + shared_weights["shared.weight"] = shared_weights["decoder.embed_tokens.weight"] + sharded_state_dicts.append(shared_weights.keys()) + + # If we only have the shared weights (dummy model/experts saved on the same file) + if len(sharded_state_dicts) == 1: + save_path = os.path.join(dump_path, weights_name) + torch.save(shared_weights, save_path) + return {weights_name: sharded_state_dicts[0]}, None + else: + torch.save(shared_weights, save_path) + # Otherwise, let's build the index + weight_map = {} + for idx, shard in enumerate(sharded_state_dicts): + shard_file = weights_name.replace(".bin", f"-{idx+1:05d}-of-{len(sharded_state_dicts):05d}.bin") + temp_filename = os.path.join(dump_path, weights_name.replace(".bin", f"-{idx+1:05d}-of-???.bin")) + os.rename(temp_filename, os.path.join(dump_path, shard_file)) + for key in shard: + weight_map[key] = shard_file + + # Add the metadata + metadata = {"total_size": total_size} + index = {"metadata": metadata, "weight_map": weight_map} + + with open(os.path.join(dump_path, WEIGHTS_INDEX_NAME), "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + return metadata, index + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # Required parameters + parser.add_argument( + "--nllb_moe_checkpoint_path", + default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000", + type=str, + required=False, + help="Path to a directory containing a folder per layer. Follows the original Google format.", + ) + parser.add_argument("--dtype", default="float32", type=str, required=False, help="dtype of the saved model") + parser.add_argument( + "--pytorch_dump_folder_path", + default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b", + type=str, + required=False, + help="Path to the output pytorch model.", + ) + args = parser.parse_args() + metadata, index = shard_on_the_fly( + args.nllb_moe_checkpoint_path, + args.pytorch_dump_folder_path, + 128, + args.dtype, + ) + + config = NllbMoeConfig.from_pretrained( + "facebook/nllb-200-3.3B", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128 + ) + config.save_pretrained(args.pytorch_dump_folder_path) + model = NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path) + print("Done") + model.save_pretrained(args.pytorch_dump_folder_path) diff --git a/src/transformers/models/nllb_moe/modeling_nllb_moe.py b/src/transformers/models/nllb_moe/modeling_nllb_moe.py new file mode 100644 index 000000000000..5bfe44e900f1 --- /dev/null +++ b/src/transformers/models/nllb_moe/modeling_nllb_moe.py @@ -0,0 +1,1843 @@ +# coding=utf-8 +# Copyright 2023 NllbMoe Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch NLLB-MoE model.""" + + +import math +import random +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn +from torch.nn import CrossEntropyLoss +from torch.utils.checkpoint import checkpoint + +from ...activations import ACT2FN +from ...deepspeed import is_deepspeed_zero3_enabled +from ...modeling_outputs import ( + MoEModelOutput, + MoEModelOutputWithPastAndCrossAttentions, + Seq2SeqMoEModelOutput, + Seq2SeqMoEOutput, +) +from ...modeling_utils import PreTrainedModel +from ...utils import ( + add_end_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_nllb_moe import NllbMoeConfig + + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "NllbMoeConfig" +_CHECKPOINT_FOR_DOC = "hf-internal-testing/dummy-nllb-moe-2-experts" +_REAL_CHECKPOINT_FOR_DOC = "facebook/nllb-moe-54b" + + +#################################################### +# This dict contains ids and associated url +# for the pretrained weights provided with the models +#################################################### +NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "facebook/nllb-moe-54b", + # See all NLLB-MOE models at https://huggingface.co/models?filter=nllb-moe +] + + +# Copied from transformers.models.bart.modeling_bart.shift_tokens_right +def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): + """ + Shift input ids one token to the right. + """ + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[:, 1:] = input_ids[:, :-1].clone() + shifted_input_ids[:, 0] = decoder_start_token_id + + if pad_token_id is None: + raise ValueError("self.model.config.pad_token_id has to be defined.") + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + return shifted_input_ids + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) + mask_cond = torch.arange(mask.size(-1)) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +# Copied from transformers.models.bart.modeling_bart._expand_mask +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +# Copied from transformers.models.roberta.modeling_roberta.create_position_ids_from_input_ids +def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0): + """ + Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols + are ignored. This is modified from fairseq's `utils.make_positions`. + + Args: + x: torch.Tensor x: + + Returns: torch.Tensor + """ + # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA. + mask = input_ids.ne(padding_idx).int() + incremental_indices = (torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length) * mask + return incremental_indices.long() + padding_idx + + +# Copied from transformers.models.switch_transformers.modeling_switch_transformers.load_balancing_loss_func with SwitchTransformers->NllbMoeModel +def load_balancing_loss_func(router_probs: torch.Tensor, expert_indices: torch.Tensor) -> float: + r""" + Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch. + + See Switch Transformer (https://arxiv.org/abs/2101.03961) for more details. This function implements the loss + function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between + experts is too unbalanced. + + Args: + router_probs (`torch.Tensor`): + Probability assigned to each expert per token. Shape: [batch_size, seqeunce_length, num_experts]. + expert_indices (`torch.Tensor`): + Indices tensor of shape [batch_size, seqeunce_length] identifying the selected expert for a given token. + + Returns: + The auxiliary loss. + """ + num_experts = router_probs.shape[-1] + + # cast the expert indices to int64, otherwise one-hot encoding will fail + if expert_indices.dtype != torch.int64: + expert_indices = expert_indices.to(torch.int64) + + if len(expert_indices.shape) == 2: + expert_indices = expert_indices.unsqueeze(2) + + expert_mask = torch.nn.functional.one_hot(expert_indices, num_experts) + + # For a given token, determine if it was routed to a given expert. + expert_mask = torch.max(expert_mask, axis=-2).values + + # cast to float32 otherwise mean will fail + expert_mask = expert_mask.to(torch.float32) + tokens_per_group_and_expert = torch.mean(expert_mask, axis=-2) + + router_prob_per_group_and_expert = torch.mean(router_probs, axis=-2) + return torch.mean(tokens_per_group_and_expert * router_prob_per_group_and_expert) * (num_experts**2) + + +# Copied from transformers.models.m2m_100.modeling_m2m_100.M2M100SinusoidalPositionalEmbedding +class NllbMoeSinusoidalPositionalEmbedding(nn.Module): + """This module produces sinusoidal positional embeddings of any length.""" + + def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None): + super().__init__() + self.offset = 2 + self.embedding_dim = embedding_dim + self.padding_idx = padding_idx + self.make_weights(num_positions + self.offset, embedding_dim, padding_idx) + + def make_weights(self, num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None): + emb_weights = self.get_embedding(num_embeddings, embedding_dim, padding_idx) + if hasattr(self, "weights"): + # in forward put the weights on the correct dtype and device of the param + emb_weights = emb_weights.to(dtype=self.weights.dtype, device=self.weights.device) + + self.register_buffer("weights", emb_weights) + + @staticmethod + def get_embedding(num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None): + """ + Build sinusoidal embeddings. + + This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of + "Attention Is All You Need". + """ + half_dim = embedding_dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb) + emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0) + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1) + if embedding_dim % 2 == 1: + # zero pad + emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1) + if padding_idx is not None: + emb[padding_idx, :] = 0 + + return emb.to(torch.get_default_dtype()) + + @torch.no_grad() + def forward( + self, input_ids: torch.Tensor = None, inputs_embeds: torch.Tensor = None, past_key_values_length: int = 0 + ): + if input_ids is not None: + bsz, seq_len = input_ids.size() + # Create the position ids from the input token ids. Any padded tokens remain padded. + position_ids = create_position_ids_from_input_ids(input_ids, self.padding_idx, past_key_values_length).to( + input_ids.device + ) + else: + bsz, seq_len = inputs_embeds.size()[:-1] + position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds, past_key_values_length) + + # expand embeddings if needed + max_pos = self.padding_idx + 1 + seq_len + past_key_values_length + if max_pos > self.weights.size(0): + self.make_weights(max_pos + self.offset, self.embedding_dim, self.padding_idx) + + return self.weights.index_select(0, position_ids.view(-1)).view(bsz, seq_len, self.weights.shape[-1]).detach() + + def create_position_ids_from_inputs_embeds(self, inputs_embeds, past_key_values_length): + """ + We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. + + Args: + inputs_embeds: torch.Tensor + + Returns: torch.Tensor + """ + input_shape = inputs_embeds.size()[:-1] + sequence_length = input_shape[1] + + position_ids = torch.arange( + self.padding_idx + 1, sequence_length + self.padding_idx + 1, dtype=torch.long, device=inputs_embeds.device + ) + return position_ids.unsqueeze(0).expand(input_shape).contiguous() + past_key_values_length + + +class NllbMoeTop2Router(nn.Module): + """ + Router using tokens choose top-2 experts assignment. + + This router uses the same mechanism as in NLLB-MoE from the fairseq repository. Items are sorted by router_probs + and then routed to their choice of expert until the expert's expert_capacity is reached. **There is no guarantee + that each token is processed by an expert**, or that each expert receives at least one token. + + The router combining weights are also returned to make sure that the states that are not updated will be masked. + + """ + + def __init__(self, config: NllbMoeConfig): + super().__init__() + self.num_experts = config.num_experts + self.expert_capacity = config.expert_capacity + self.classifier = nn.Linear(config.hidden_size, self.num_experts, bias=config.router_bias) + self.router_ignore_padding_tokens = config.router_ignore_padding_tokens + self.dtype = getattr(torch, config.router_dtype) + + self.second_expert_policy = config.second_expert_policy + self.normalize_router_prob_before_dropping = config.normalize_router_prob_before_dropping + self.batch_prioritized_routing = config.batch_prioritized_routing + self.moe_eval_capacity_token_fraction = config.moe_eval_capacity_token_fraction + + def _cast_classifier(self): + r""" + `bitsandbytes` `Linear8bitLt` layers does not support manual casting Therefore we need to check if they are an + instance of the `Linear8bitLt` class by checking special attributes. + """ + if not (hasattr(self.classifier, "SCB") or hasattr(self.classifier, "CB")): + self.classifier = self.classifier.to(self.dtype) + + def normalize_router_probabilities(self, router_probs, top_1_mask, top_2_mask): + top_1_max_probs = (router_probs * top_1_mask).sum(dim=1) + top_2_max_probs = (router_probs * top_2_mask).sum(dim=1) + denom_s = torch.clamp(top_1_max_probs + top_2_max_probs, min=torch.finfo(router_probs.dtype).eps) + top_1_max_probs = top_1_max_probs / denom_s + top_2_max_probs = top_2_max_probs / denom_s + return top_1_max_probs, top_2_max_probs + + def route_tokens( + self, + router_logits: torch.Tensor, + input_dtype: torch.dtype = torch.float32, + padding_mask: Optional[torch.LongTensor] = None, + ) -> Tuple: + """ + Computes the `dispatch_mask` and the `dispatch_weights` for each experts. The masks are adapted to the expert + capacity. + """ + nb_tokens = router_logits.shape[0] + # Apply Softmax and cast back to the original `dtype` + router_probs = nn.functional.softmax(router_logits, dim=-1, dtype=self.dtype).to(input_dtype) + top_1_expert_index = torch.argmax(router_probs, dim=-1) + top_1_mask = torch.nn.functional.one_hot(top_1_expert_index, num_classes=self.num_experts) + + if self.second_expert_policy == "sampling": + gumbel = torch.distributions.gumbel.Gumbel(0, 1).rsample + router_logits += gumbel(router_logits.shape).to(router_logits.device) + + # replace top_1_expert_index with min values + logits_except_top_1 = router_logits.masked_fill(top_1_mask.bool(), float("-inf")) + top_2_expert_index = torch.argmax(logits_except_top_1, dim=-1) + top_2_mask = torch.nn.functional.one_hot(top_2_expert_index, num_classes=self.num_experts) + + if self.normalize_router_prob_before_dropping: + top_1_max_probs, top_2_max_probs = self.normalize_router_probabilities( + router_probs, top_1_mask, top_2_mask + ) + + if self.second_expert_policy == "random": + top_2_max_probs = (router_probs * top_2_mask).sum(dim=1) + sampled = (2 * top_2_max_probs) > torch.rand_like(top_2_max_probs.float()) + top_2_mask = top_2_mask * sampled.repeat(self.num_experts, 1).transpose(1, 0) + + if padding_mask is not None and not self.router_ignore_padding_tokens: + if len(padding_mask.shape) == 4: + # only get the last causal mask + padding_mask = padding_mask[:, :, -1, :].reshape(-1)[-nb_tokens:] + non_padding = ~padding_mask.bool() + top_1_mask = top_1_mask * non_padding.unsqueeze(-1).to(top_1_mask.dtype) + top_2_mask = top_2_mask * non_padding.unsqueeze(-1).to(top_1_mask.dtype) + + if self.batch_prioritized_routing: + # sort tokens based on their routing probability + # to make sure important tokens are routed, first + importance_scores = -1 * router_probs.max(dim=1)[0] + sorted_top_1_mask = top_1_mask[importance_scores.argsort(dim=0)] + sorted_cumsum1 = (torch.cumsum(sorted_top_1_mask, dim=0) - 1) * sorted_top_1_mask + locations1 = sorted_cumsum1[importance_scores.argsort(dim=0).argsort(dim=0)] + + sorted_top_2_mask = top_2_mask[importance_scores.argsort(dim=0)] + sorted_cumsum2 = (torch.cumsum(sorted_top_2_mask, dim=0) - 1) * sorted_top_2_mask + locations2 = sorted_cumsum2[importance_scores.argsort(dim=0).argsort(dim=0)] + # Update 2nd's location by accounting for locations of 1st + locations2 += torch.sum(top_1_mask, dim=0, keepdim=True) + + else: + locations1 = torch.cumsum(top_1_mask, dim=0) - 1 + locations2 = torch.cumsum(top_2_mask, dim=0) - 1 + # Update 2nd's location by accounting for locations of 1st + locations2 += torch.sum(top_1_mask, dim=0, keepdim=True) + + if not self.training and self.moe_eval_capacity_token_fraction > 0: + self.expert_capacity = math.ceil(self.moe_eval_capacity_token_fraction * nb_tokens) + else: + capacity = 2 * math.ceil(nb_tokens / self.num_experts) + self.expert_capacity = capacity if self.expert_capacity is None else self.expert_capacity + + # Remove locations outside capacity from ( cumsum < capacity = False will not be routed) + top_1_mask = top_1_mask * torch.lt(locations1, self.expert_capacity) + top_2_mask = top_2_mask * torch.lt(locations2, self.expert_capacity) + + if not self.normalize_router_prob_before_dropping: + top_1_max_probs, top_2_max_probs = self.normalize_router_probabilities( + router_probs, top_1_mask, top_2_mask + ) + + # Calculate combine_weights and dispatch_mask + gates1 = top_1_max_probs[:, None] * top_1_mask + gates2 = top_2_max_probs[:, None] * top_2_mask + router_probs = gates1 + gates2 + + return top_1_mask, router_probs + + def forward(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.LongTensor] = None) -> Tuple: + r""" + The hidden states are reshaped to simplify the computation of the router probabilities (combining weights for + each experts.) + + Args: + hidden_states (`torch.Tensor`): + (batch_size, sequence_length, hidden_dim) from which router probabilities are computed. + Returns: + top_1_mask (`torch.Tensor` of shape (batch_size, sequence_length)): + Index tensor of shape [batch_size, sequence_length] corresponding to the expert selected for each token + using the top1 probabilities of the router. + router_probabilities (`torch.Tensor` of shape (batch_size, sequence_length, nump_experts)): + Tensor of shape (batch_size, sequence_length, num_experts) corresponding to the probabilities for each + token and expert. Used for routing tokens to experts. + router_logits (`torch.Tensor` of shape (batch_size, sequence_length))): + Logits tensor of shape (batch_size, sequence_length, num_experts) corresponding to raw router logits. + This is used later for computing router z-loss. + """ + self.input_dtype = hidden_states.dtype + batch_size, sequence_length, hidden_dim = hidden_states.shape + hidden_states = hidden_states.reshape((batch_size * sequence_length), hidden_dim) + hidden_states = hidden_states.to(self.dtype) + self._cast_classifier() + router_logits = self.classifier(hidden_states) + top_1_mask, router_probs = self.route_tokens(router_logits, self.input_dtype, padding_mask) + return top_1_mask, router_probs + + +class NllbMoeDenseActDense(nn.Module): + def __init__(self, config: NllbMoeConfig, ffn_dim: int): + super().__init__() + self.fc1 = nn.Linear(config.d_model, ffn_dim) + self.fc2 = nn.Linear(ffn_dim, config.d_model) + self.dropout = nn.Dropout(config.activation_dropout) + self.act = ACT2FN[config.activation_function] + + def forward(self, hidden_states): + hidden_states = self.fc1(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.dropout(hidden_states) + if ( + isinstance(self.fc2.weight, torch.Tensor) + and hidden_states.dtype != self.fc2.weight.dtype + and self.fc2.weight.dtype != torch.int8 + ): + hidden_states = hidden_states.to(self.fc2.weight.dtype) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +class NllbMoeSparseMLP(nn.Module): + r""" + Implementation of the NLLB-MoE sparse MLP module. + """ + + def __init__(self, config: NllbMoeConfig, ffn_dim: int, expert_class: nn.Module = NllbMoeDenseActDense): + super().__init__() + self.router = NllbMoeTop2Router(config) + self.moe_token_dropout = config.moe_token_dropout + self.token_dropout = nn.Dropout(self.moe_token_dropout) + self.num_experts = config.num_experts + + self.experts = nn.ModuleDict() + for idx in range(self.num_experts): + self.experts[f"expert_{idx}"] = expert_class(config, ffn_dim) + + def forward(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.Tensor] = False): + r""" + The goal of this forward pass is to have the same number of operation as the equivalent `NllbMoeDenseActDense` + (mlp) layer. This means that all of the hidden states should be processed at most twice ( since we are using a + top_2 gating mecanism). This means that we keep the complexity to O(batch_size x sequence_length x hidden_dim) + instead of O(num_experts x batch_size x sequence_length x hidden_dim). + + 1- Get the `router_probs` from the `router`. The shape of the `router_mask` is `(batch_size X sequence_length, + num_expert)` and corresponds to the boolean version of the `router_probs`. The inputs are masked using the + `router_mask`. + + 2- Dispatch the hidden_states to its associated experts. The router probabilities are used to weight the + contribution of each experts when updating the masked hidden states. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, sequence_length, hidden_dim)`): + The hidden states + padding_mask (`torch.Tensor`, *optional*, defaults to `False`): + Attention mask. Can be in the causal form or not. + + Returns: + hidden_states (`torch.Tensor` of shape `(batch_size, sequence_lenght, hidden_dim)`): + Updated hidden states + router_logits (`torch.Tensor` of shape `(batch_size, sequence_length, num_experts)`): + Needed for computing the loss + + """ + batch_size, sequence_length, hidden_dim = hidden_states.shape + + top_1_mask, router_probs = self.router(hidden_states, padding_mask) + router_mask = router_probs.bool() + hidden_states = hidden_states.reshape((batch_size * sequence_length), hidden_dim) + masked_hidden_states = torch.einsum("bm,be->ebm", hidden_states, router_mask) + for idx, expert in enumerate(self.experts.values()): + token_indices = router_mask[:, idx] + combining_weights = router_probs[token_indices, idx] + expert_output = expert(masked_hidden_states[idx, token_indices]) + if self.moe_token_dropout > 0: + if self.training: + expert_output = self.token_dropout(expert_output) + else: + expert_output *= 1 - self.moe_token_dropout + masked_hidden_states[idx, token_indices] = torch.einsum("b,be->be", combining_weights, expert_output) + hidden_states = masked_hidden_states.sum(dim=0).reshape(batch_size, sequence_length, hidden_dim) + + top_1_expert_index = torch.argmax(top_1_mask, dim=-1) + return hidden_states, (router_probs, top_1_expert_index) + + +# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->NllbMoe,key_value_states->encoder_hidden_states +class NllbMoeAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + is_decoder: bool = False, + bias: bool = True, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if encoder_hidden_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = encoder_hidden_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + # `past_key_value[0].shape[2] == encoder_hidden_states.shape[1]` + # is checking that the `sequence_length` of the `past_key_value` is the same as + # the provided `encoder_hidden_states` to support prefix tuning + if ( + is_cross_attention + and past_key_value is not None + and past_key_value[0].shape[2] == encoder_hidden_states.shape[1] + ): + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(encoder_hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(encoder_hidden_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.reshape(*proj_shape) + value_states = value_states.reshape(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" + f" {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned across GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value + + +class NllbMoeEncoderLayer(nn.Module): + def __init__(self, config: NllbMoeConfig, is_sparse: bool = False): + super().__init__() + self.embed_dim = config.d_model + self.is_sparse = is_sparse + self.self_attn = NllbMoeAttention( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + dropout=config.attention_dropout, + ) + self.attn_dropout = nn.Dropout(config.dropout) + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + if not self.is_sparse: + self.ffn = NllbMoeDenseActDense(config, ffn_dim=config.encoder_ffn_dim) + else: + self.ffn = NllbMoeSparseMLP(config, ffn_dim=config.encoder_ffn_dim) + self.ff_layer_norm = nn.LayerNorm(config.d_model) + self.ff_dropout = nn.Dropout(config.activation_dropout) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + layer_head_mask: torch.Tensor, + output_attentions: bool = False, + output_router_logits: bool = False, + ) -> torch.Tensor: + """ + Args: + hidden_states (`torch.FloatTensor`): + input to the layer of shape `(seq_len, batch, embed_dim)` + attention_mask (`torch.FloatTensor`): + attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very + large negative values. + layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size + `(encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states, attn_weights, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = self.attn_dropout(hidden_states) + hidden_states = residual + hidden_states + + residual = hidden_states + + hidden_states = self.ff_layer_norm(hidden_states) + if self.is_sparse: + hidden_states, router_states = self.ffn(hidden_states, attention_mask) + else: + hidden_states = self.ffn(hidden_states) + hidden_states = self.ff_dropout(hidden_states) + + hidden_states = residual + hidden_states + + if hidden_states.dtype == torch.float16 and ( + torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any() + ): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + if output_router_logits: + outputs += (router_states,) + + return outputs + + +class NllbMoeDecoderLayer(nn.Module): + def __init__(self, config: NllbMoeConfig, is_sparse: bool = False): + super().__init__() + self.embed_dim = config.d_model + self.is_sparse = is_sparse + self.self_attn = NllbMoeAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + is_decoder=True, + ) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.attn_dropout = nn.Dropout(config.dropout) + + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.cross_attention = NllbMoeAttention( + self.embed_dim, config.decoder_attention_heads, config.attention_dropout, is_decoder=True + ) + self.cross_attention_layer_norm = nn.LayerNorm(self.embed_dim) + if not self.is_sparse: + self.ffn = NllbMoeDenseActDense(config, ffn_dim=config.decoder_ffn_dim) + else: + self.ffn = NllbMoeSparseMLP(config, ffn_dim=config.decoder_ffn_dim) + self.ff_layer_norm = nn.LayerNorm(config.d_model) + self.ff_dropout = nn.Dropout(config.activation_dropout) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + cross_attn_layer_head_mask: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + output_router_logits: Optional[bool] = False, + use_cache: Optional[bool] = True, + ) -> torch.Tensor: + """ + Args: + hidden_states (`torch.FloatTensor`): + input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): + attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very + large negative values. + encoder_hidden_states (`torch.FloatTensor`): + cross attention input to the layer of shape `(batch, seq_len, embed_dim)` + encoder_attention_mask (`torch.FloatTensor`): + encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by + very large negative values. + layer_head_mask (`torch.FloatTensor`): + mask for attention heads in a given layer of size `(encoder_attention_heads,)`. + cross_attn_layer_head_mask (`torch.FloatTensor`): + mask for cross-attention heads in a given layer of size `(decoder_attention_heads,)`. + past_key_value (`Tuple(torch.FloatTensor)`): + cached past key and value projection states + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Self Attention + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + # add present self-attn cache to positions 1,2 of present_key_value tuple + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + past_key_value=self_attn_past_key_value, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = self.attn_dropout(hidden_states) + hidden_states = residual + hidden_states + + # Cross-Attention Block + cross_attn_present_key_value = None + cross_attn_weights = None + if encoder_hidden_states is not None: + residual = hidden_states + hidden_states = self.cross_attention_layer_norm(hidden_states) + + # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple + cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None + hidden_states, cross_attn_weights, cross_attn_present_key_value = self.cross_attention( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + past_key_value=cross_attn_past_key_value, + attention_mask=encoder_attention_mask, + layer_head_mask=cross_attn_layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = self.attn_dropout(hidden_states) + hidden_states = residual + hidden_states + + # add cross-attn to positions 3,4 of present_key_value tuple + present_key_value += cross_attn_present_key_value + + # Fully Connected + residual = hidden_states + + hidden_states = self.ff_layer_norm(hidden_states) + if self.is_sparse: + hidden_states, router_states = self.ffn(hidden_states, attention_mask) + else: + hidden_states = self.ffn(hidden_states) + hidden_states = self.ff_dropout(hidden_states) + + hidden_states = residual + hidden_states + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any(): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states, present_key_value) + + if output_attentions: + outputs += (self_attn_weights, cross_attn_weights) + + if output_router_logits: + outputs += (router_states,) + + return outputs + + +class NllbMoePreTrainedModel(PreTrainedModel): + config_class = NllbMoeConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["NllbMoeAttention"] + + def _init_weights(self, module): + """Initialize the weights""" + std = self.config.init_std + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (NllbMoeDecoder, NllbMoeEncoder)): + module.gradient_checkpointing = value + + +NLLB_MOE_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`NllbMoeConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +NLLB_MOE_GENERATION_EXAMPLE = r""" + Translation example: + + ```python + >>> from transformers import AutoTokenizer, NllbMoeForConditionalGeneration + + >>> model = NllbMoeForConditionalGeneration.from_pretrained("facebook/nllb-moe-54b") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/nllb-moe-54b") + + >>> text_to_translate = "Life is like a box of chocolates" + >>> model_inputs = tokenizer(text_to_translate, return_tensors="pt") + + >>> # translate to French + >>> gen_tokens = model.generate(**model_inputs, forced_bos_token_id=tokenizer.get_lang_id("eng_Latn")) + >>> print(tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)) + ``` +""" + +NLLB_MOE_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + NllbMoe uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If + `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape + `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you + can choose to directly pass an embedded representation. This is useful if you want more control over how to + convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more control over how to convert + `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + + If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value + of `inputs_embeds`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + output_router_logits (`bool`, *optional*): + Whether or not to return the logits of all the routers. They are useful for computing the router loss, and + should not be returned during inference. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +class NllbMoeEncoder(NllbMoePreTrainedModel): + """ + Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a + [`NllbMoeEncoderLayer`]. + + Args: + config: + NllbMoeConfig + embed_tokens (nn.Embedding): + output embedding + """ + + def __init__(self, config: NllbMoeConfig, embed_tokens: Optional[nn.Embedding] = None): + super().__init__(config) + + self.dropout = config.dropout + self.layerdrop = config.encoder_layerdrop + + embed_dim = config.d_model + self.padding_idx = config.pad_token_id + self.max_source_positions = config.max_position_embeddings + self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0 + + self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim, self.padding_idx) + + if embed_tokens is not None: + self.embed_tokens.weight = embed_tokens.weight + + self.embed_positions = NllbMoeSinusoidalPositionalEmbedding( + config.max_position_embeddings, + embed_dim, + self.padding_idx, + ) + sparse_step = config.encoder_sparse_step + self.layers = nn.ModuleList() + for i in range(config.encoder_layers): + is_sparse = (i + 1) % sparse_step == 0 if sparse_step > 0 else False + self.layers.append(NllbMoeEncoderLayer(config, is_sparse)) + + self.layer_norm = nn.LayerNorm(config.d_model) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + output_router_logits (`bool`, *optional*): + Whether or not to return the logits of all the routers. They are useful for computing the router loss, + and should not be returned during inference. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + + embed_pos = self.embed_positions(input_ids, inputs_embeds) + embed_pos = embed_pos.to(inputs_embeds.device) + + hidden_states = inputs_embeds + embed_pos + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + # expand attention_mask + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(attention_mask, inputs_embeds.dtype) + + encoder_states = () if output_hidden_states else None + all_router_probs = () if output_router_logits else None + all_attentions = () if output_attentions else None + + # check if head_mask has a correct number of layers specified if desired + if head_mask is not None: + if head_mask.size()[0] != len(self.layers): + raise ValueError( + f"The head_mask should be specified for {len(self.layers)} layers, but it is for" + f" {head_mask.size()[0]}." + ) + + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + dropout_probability = random.uniform(0, 1) + if self.training and (dropout_probability < self.layerdrop): # skip the layer + layer_outputs = (None, None, None) + else: + if self.gradient_checkpointing and self.training: + # create gradient checkpointing function + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(encoder_layer), + hidden_states, + attention_mask, + (head_mask[idx] if head_mask is not None else None), + ) + else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask, + layer_head_mask=(head_mask[idx] if head_mask is not None else None), + output_attentions=output_attentions, + output_router_logits=output_router_logits, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions += (layer_outputs[1],) + + if output_router_logits: + all_router_probs += (layer_outputs[-1],) + + last_hidden_state = self.layer_norm(hidden_states) + + if output_hidden_states: + encoder_states += (last_hidden_state,) + + if not return_dict: + return tuple( + v for v in [last_hidden_state, encoder_states, all_attentions, all_router_probs] if v is not None + ) + + return MoEModelOutput( + last_hidden_state=last_hidden_state, + hidden_states=encoder_states, + attentions=all_attentions, + router_probs=all_router_probs, + ) + + +class NllbMoeDecoder(NllbMoePreTrainedModel): + """ + Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`NllbMoeDecoderLayer`] + + Args: + config: + NllbMoeConfig + embed_tokens (nn.Embedding): + output embedding + """ + + def __init__(self, config: NllbMoeConfig, embed_tokens: Optional[nn.Embedding] = None): + super().__init__(config) + self.dropout = config.dropout + self.layerdrop = config.decoder_layerdrop + self.padding_idx = config.pad_token_id + self.max_target_positions = config.max_position_embeddings + self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 + + self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, self.padding_idx) + + if embed_tokens is not None: + self.embed_tokens.weight = embed_tokens.weight + + self.embed_positions = NllbMoeSinusoidalPositionalEmbedding( + config.max_position_embeddings, + config.d_model, + self.padding_idx, + ) + + sparse_step = config.decoder_sparse_step + self.layers = nn.ModuleList() + for i in range(config.decoder_layers): + is_sparse = (i + 1) % sparse_step == 0 if sparse_step > 0 else False + self.layers.append(NllbMoeDecoderLayer(config, is_sparse)) + + self.layer_norm = nn.LayerNorm(config.d_model) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing + cross-attention on hidden heads. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of + shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing + `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more + control over how to convert `input_ids` indices into associated vectors than the model's internal + embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + output_router_logits (`bool`, *optional*): + Whether or not to return the logits of all the routers. They are useful for computing the router loss, + and should not be returned during inference. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length + ).to(inputs_embeds.device) + + if attention_mask is not None and combined_attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = combined_attention_mask + _expand_mask( + attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1] + ) + + # expand encoder attention mask + if encoder_hidden_states is not None and encoder_attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + encoder_attention_mask = _expand_mask(encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]) + + # embed positions + positions = self.embed_positions(input_ids, inputs_embeds, past_key_values_length) + positions = positions.to(inputs_embeds.device) + + hidden_states = inputs_embeds + positions + + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting" " `use_cache=False`..." + ) + use_cache = False + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_router_probs = () if output_router_logits else None + all_cross_attentions = () if output_attentions else None + present_key_value_states = () if use_cache else None + + # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired + for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]): + if attn_mask is not None: + if attn_mask.size()[0] != len(self.layers): + raise ValueError( + f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for" + f" {head_mask.size()[0]}." + ) + deepspeed_zero3_is_enabled = is_deepspeed_zero3_enabled() + + for idx, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + dropout_probability = random.uniform(0, 1) + + skip_the_layer = True if self.training and (dropout_probability < self.layerdrop) else False + if not skip_the_layer or deepspeed_zero3_is_enabled: + layer_head_mask = head_mask[idx] if head_mask is not None else None + cross_attn_layer_head_mask = cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + # under deepspeed zero3 all gpus must run in sync + if self.gradient_checkpointing and self.training: + if use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + def create_custom_forward(module): + def custom_forward(*inputs): + return tuple(module(*inputs, use_cache, output_attentions)) + + return custom_forward + + layer_outputs = checkpoint( + create_custom_forward(decoder_layer), + hidden_states, + combined_attention_mask, + encoder_hidden_states, + encoder_attention_mask, + layer_head_mask, + cross_attn_layer_head_mask, + None, # past_key_value is always None with gradient checkpointing + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=combined_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + layer_head_mask=layer_head_mask, + cross_attn_layer_head_mask=cross_attn_layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + output_router_logits=output_router_logits, + ) + + hidden_states = layer_outputs[0] + + if skip_the_layer: + continue + + if use_cache: + present_key_value_states += (layer_outputs[1],) + + if output_attentions: + all_self_attns += (layer_outputs[2],) + all_cross_attentions += (layer_outputs[3],) + + if output_router_logits: + all_router_probs += (layer_outputs[-1],) + + hidden_states = self.layer_norm(hidden_states) + + # Add last layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + present_key_value_states, + all_hidden_states, + all_self_attns, + all_cross_attentions, + all_router_probs, + ] + if v is not None + ) + return MoEModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=present_key_value_states, + hidden_states=all_hidden_states, + attentions=all_self_attns, + cross_attentions=all_cross_attentions, + router_probs=all_router_probs, + ) + + +@add_start_docstrings( + "The bare NllbMoe Model outputting raw hidden-states without any specific head on top.", + NLLB_MOE_START_DOCSTRING, +) +class NllbMoeModel(NllbMoePreTrainedModel): + _keys_to_ignore_on_load_missing = [ + "encoder.embed_tokens.weight", + "decoder.embed_tokens.weight", + "encoder.embed_positions.weights", + "encoder.embed_positions.bias", + "decoder.embed_positions.weights", + "decoder.embed_positions.bias", + ] + + def __init__(self, config: NllbMoeConfig): + super().__init__(config) + + padding_idx, vocab_size = config.pad_token_id, config.vocab_size + self.shared = nn.Embedding(vocab_size, config.d_model, padding_idx) + + self.encoder = NllbMoeEncoder(config, self.shared) + self.decoder = NllbMoeDecoder(config, self.shared) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, value): + self.shared = value + self.encoder.embed_tokens = self.shared + self.decoder.embed_tokens = self.shared + + def get_encoder(self): + return self.encoder + + def get_decoder(self): + return self.decoder + + @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING) + @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Seq2SeqMoEModelOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], Seq2SeqMoEModelOutput]: + r""" + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, NllbMoeModel + + >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/random-nllb-moe-2-experts") + >>> model = SwitchTransformersModel.from_pretrained("hf-internal-testing/random-nllb-moe-2-experts") + + >>> input_ids = tokenizer( + ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" + ... ).input_ids # Batch size 1 + >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 + + >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for NllbMoeModel + >>> decoder_input_ids = model._shift_right(decoder_input_ids) + + >>> # forward pass + >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) + >>> last_hidden_states = outputs.last_hidden_state + ```""" + return_dict = return_dict if return_dict is not None else self.config.return_dict + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, + return_dict=return_dict, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, MoEModelOutput): + encoder_outputs = MoEModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + router_probs=encoder_outputs[3] if len(encoder_outputs) > 3 else None, + ) + + # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_outputs[0], + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, + return_dict=return_dict, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return Seq2SeqMoEModelOutput( + past_key_values=decoder_outputs.past_key_values, + cross_attentions=decoder_outputs.cross_attentions, + last_hidden_state=decoder_outputs.last_hidden_state, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + decoder_hidden_states=decoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + decoder_attentions=decoder_outputs.attentions, + encoder_router_logits=encoder_outputs.router_probs, + decoder_router_logits=decoder_outputs.router_probs, + ) + + +@add_start_docstrings( + "The NllbMoe Model with a language modeling head. Can be used for summarization.", NLLB_MOE_START_DOCSTRING +) +class NllbMoeForConditionalGeneration(NllbMoePreTrainedModel): + base_model_prefix = "model" + _keys_to_ignore_on_load_missing = [ + r"encoder.version", + r"decoder.version", + r"lm_head.weight", + r"encoder.embed_tokens.weight", + r"decoder.embed_tokens.weight", + r"encoder.embed_positions.weights", + r"encoder.embed_positions.bias", + r"decoder.embed_positions.weights", + r"decoder.embed_positions.bias", + ] + + def __init__(self, config: NllbMoeConfig): + super().__init__(config) + self.model = NllbMoeModel(config) + self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) + + self.router_z_loss_coef = config.router_z_loss_coef + self.router_aux_loss_coef = config.router_aux_loss_coef + # Initialize weights and apply final processing + self.post_init() + + def get_encoder(self): + return self.model.get_encoder() + + def get_decoder(self): + return self.model.get_decoder() + + def resize_token_embeddings(self, new_num_tokens: int) -> nn.Embedding: + new_embeddings = super().resize_token_embeddings(new_num_tokens) + return new_embeddings + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Seq2SeqMoEOutput, config_class=_CONFIG_FOR_DOC) + @add_end_docstrings(NLLB_MOE_GENERATION_EXAMPLE) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + output_router_logits: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.Tensor], Seq2SeqMoEOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + """ + return_dict = return_dict if return_dict is not None else self.config.return_dict + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_router_logits = ( + output_router_logits if output_router_logits is not None else self.config.output_router_logits + ) + if labels is not None: + if decoder_input_ids is None: + decoder_input_ids = shift_tokens_right( + labels, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + encoder_outputs=encoder_outputs, + decoder_attention_mask=decoder_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + output_router_logits=output_router_logits, + return_dict=return_dict, + ) + lm_logits = self.lm_head(outputs[0]) + + loss = None + encoder_aux_loss = None + decoder_aux_loss = None + + if labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-100) + # todo check in the config if router loss enables + + if output_router_logits: + encoder_router_logits = outputs[-1] + decoder_router_logits = outputs[5 if output_attentions else 3] + + # Compute the router loss (z_loss + auxiliary loss) for each router in the encoder and decoder + encoder_router_logits, encoder_expert_indexes = self._unpack_router_logits(encoder_router_logits) + encoder_aux_loss = load_balancing_loss_func(encoder_router_logits, encoder_expert_indexes) + + decoder_router_logits, decoder_expert_indexes = self._unpack_router_logits(decoder_router_logits) + decoder_aux_loss = load_balancing_loss_func(decoder_router_logits, decoder_expert_indexes) + + loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1)) + + if output_router_logits and labels is not None: + aux_loss = self.router_aux_loss_coef * (encoder_aux_loss + decoder_aux_loss) + loss = loss + aux_loss + + output = (loss,) if loss is not None else () + if not return_dict: + output += (lm_logits,) + if output_router_logits: # only return the loss if they are not None + output += ( + encoder_aux_loss, + decoder_aux_loss, + *outputs[1:], + ) + else: + output += outputs[1:] + + return output + + return Seq2SeqMoEOutput( + loss=loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, + cross_attentions=outputs.cross_attentions, + encoder_aux_loss=encoder_aux_loss, + decoder_aux_loss=decoder_aux_loss, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + decoder_hidden_states=outputs.decoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + decoder_attentions=outputs.decoder_attentions, + encoder_router_logits=outputs.encoder_router_logits, + decoder_router_logits=outputs.decoder_router_logits, + ) + + # Copied from transfomers.models.switch_transformers.SwitchTransformersForConditionalGeneration._unpack_router_logits + def _unpack_router_logits(self, router_outputs): + total_router_logits = [] + total_expert_indexes = [] + for router_output in router_outputs: + if router_output is not None: + router_logits, expert_indexes = router_output + total_router_logits.append(router_logits) + total_expert_indexes.append(expert_indexes) + if len(total_expert_indexes) > 0: + total_router_logits = torch.cat(total_router_logits, dim=1) + if len(total_expert_indexes) > 0: + torch.cat(total_expert_indexes, dim=1) + return torch.cat(total_router_logits, dim=1), torch.cat(total_expert_indexes, dim=1) + + # Copied from transfomers.models.switch_transformers.SwitchTransformersForConditionalGeneration.prepare_inputs_for_generation + def prepare_inputs_for_generation( + self, + decoder_input_ids, + past_key_values=None, + attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, + use_cache=None, + encoder_outputs=None, + **kwargs, + ): + # cut decoder_input_ids if past is used + if past_key_values is not None: + decoder_input_ids = decoder_input_ids[:, -1:] + + return { + "input_ids": None, # encoder_outputs is defined. input_ids not needed + "encoder_outputs": encoder_outputs, + "past_key_values": past_key_values, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "head_mask": head_mask, + "decoder_head_mask": decoder_head_mask, + "cross_attn_head_mask": cross_attn_head_mask, + "use_cache": use_cache, # change this to avoid caching (presumably for debugging) + } + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past diff --git a/src/transformers/models/switch_transformers/configuration_switch_transformers.py b/src/transformers/models/switch_transformers/configuration_switch_transformers.py index dd6c6c03e2cf..16cc51c6739e 100644 --- a/src/transformers/models/switch_transformers/configuration_switch_transformers.py +++ b/src/transformers/models/switch_transformers/configuration_switch_transformers.py @@ -111,7 +111,6 @@ def __init__( num_sparse_decoder_layers=3, num_heads=12, num_experts=8, - router_type="tokens_masked", router_bias=False, router_jitter_noise=0.01, router_dtype="float32", @@ -157,7 +156,6 @@ def __init__( self.decoder_sparse_step = self.num_decoder_layers # HACK: this will create 0 sparse layers self.num_heads = num_heads - self.router_type = router_type self.num_experts = num_experts self.expert_capacity = expert_capacity self.router_bias = router_bias diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index f016538fac68..4859974f22b2 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -4782,6 +4782,44 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class NllbMoeForConditionalGeneration(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class NllbMoeModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class NllbMoePreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class NllbMoeSparseMLP(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class NllbMoeTop2Router(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/tests/models/nllb_moe/__init__.py b/tests/models/nllb_moe/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/nllb_moe/test_modeling_nllb_moe.py b/tests/models/nllb_moe/test_modeling_nllb_moe.py new file mode 100644 index 000000000000..e2fb1483f7f8 --- /dev/null +++ b/tests/models/nllb_moe/test_modeling_nllb_moe.py @@ -0,0 +1,556 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch NLLB-MoE model. """ + + +import copy +import tempfile +import unittest + +from transformers import NllbMoeConfig, is_torch_available, set_seed +from transformers.testing_utils import ( + require_sentencepiece, + require_tokenizers, + require_torch, + slow, + torch_device, +) +from transformers.utils import cached_property + +from ...generation.test_utils import GenerationTesterMixin +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, ids_tensor +from ...test_pipeline_mixin import PipelineTesterMixin + + +if is_torch_available(): + import torch + + from transformers import NllbMoeForConditionalGeneration, NllbMoeModel, NllbTokenizer + from transformers.models.nllb_moe.modeling_nllb_moe import NllbMoeDecoder, NllbMoeEncoder, NllbMoeTop2Router + + +class NllbMoeModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_labels=False, + vocab_size=99, + hidden_size=16, + num_hidden_layers=4, + num_attention_heads=4, + intermediate_size=4, + hidden_act="relu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + encoder_layerdrop=0.0, + decoder_layerdrop=0.0, + max_position_embeddings=20, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + num_experts=4, + encoder_sparse_step=2, + decoder_sparse_step=1, + expert_capacity=100, + router_jitter_noise=0.0, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.encoder_layerdrop = encoder_layerdrop + self.decoder_layerdrop = decoder_layerdrop + self.max_position_embeddings = max_position_embeddings + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.encoder_sparse_step = encoder_sparse_step + self.decoder_sparse_step = decoder_sparse_step + self.expert_capacity = expert_capacity + self.router_jitter_noise = router_jitter_noise + self.num_experts = num_experts + + def prepare_nllb_moe_inputs_dict( + self, + config, + input_ids, + decoder_input_ids, + attention_mask=None, + decoder_attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, + ): + if attention_mask is None: + attention_mask = input_ids.ne(config.pad_token_id) + if decoder_attention_mask is None: + decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id) + if head_mask is None: + head_mask = torch.ones(config.encoder_layers, config.encoder_attention_heads, device=torch_device) + if decoder_head_mask is None: + decoder_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device) + if cross_attn_head_mask is None: + cross_attn_head_mask = torch.ones( + config.decoder_layers, config.decoder_attention_heads, device=torch_device + ) + return { + "input_ids": input_ids, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "decoder_attention_mask": attention_mask, + "head_mask": head_mask, + "decoder_head_mask": decoder_head_mask, + "cross_attn_head_mask": cross_attn_head_mask, + } + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_ids[:, -1] = self.eos_token_id # Eos Token + decoder_input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + # we need to clamp the input ids here to avoid having pad token in between + # this is because for NllbMoe the position_ids are prepared such that + # all pad tokens have pos id = 2 and rest are between 2..seq_length + # and the seq_length here is seq_length - num_pad_tokens + # but when using past, there is no way of knowing if the past input ids had + # pad tokens in them, which results in incorrect seq_lenth and which in turn results in + # position_ids being off by num_pad_tokens in past input + input_ids = input_ids.clamp(self.pad_token_id + 1) + decoder_input_ids = decoder_input_ids.clamp(self.pad_token_id + 1) + + config = self.get_config() + inputs_dict = self.prepare_nllb_moe_inputs_dict(config, input_ids, decoder_input_ids) + return config, inputs_dict + + def get_config(self): + return NllbMoeConfig( + vocab_size=self.vocab_size, + d_model=self.hidden_size, + encoder_layers=self.num_hidden_layers, + decoder_layers=self.num_hidden_layers, + encoder_attention_heads=self.num_attention_heads, + decoder_attention_heads=self.num_attention_heads, + encoder_ffn_dim=self.intermediate_size, + decoder_ffn_dim=self.intermediate_size, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + encoder_layerdrop=self.encoder_layerdrop, + decoder_layerdrop=self.decoder_layerdrop, + max_position_embeddings=self.max_position_embeddings, + eos_token_id=self.eos_token_id, + bos_token_id=self.bos_token_id, + pad_token_id=self.pad_token_id, + expert_capacity=self.expert_capacity, + router_jitter_noise=self.router_jitter_noise, + decoder_sparse_step=self.decoder_sparse_step, + encoder_sparse_step=self.encoder_sparse_step, + num_experts=self.num_experts, + ) + + def prepare_config_and_inputs_for_common(self): + config, inputs_dict = self.prepare_config_and_inputs() + return config, inputs_dict + + @require_torch + def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): + model = NllbMoeModel(config=config).get_decoder().to(torch_device).eval() + input_ids = inputs_dict["input_ids"] + attention_mask = inputs_dict["attention_mask"] + head_mask = inputs_dict["head_mask"] + + # first forward pass + outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True) + + output, past_key_values = outputs.to_tuple() + + # create hypothetical multiple next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) + next_attn_mask = ids_tensor((self.batch_size, 3), 2) + + # append to next input_ids and + next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) + next_attention_mask = torch.cat([attention_mask, next_attn_mask], dim=-1) + + output_from_no_past = model(next_input_ids, attention_mask=next_attention_mask)["last_hidden_state"] + output_from_past = model(next_tokens, attention_mask=next_attention_mask, past_key_values=past_key_values)[ + "last_hidden_state" + ] + + # select random slice + random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() + output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach() + output_from_past_slice = output_from_past[:, :, random_slice_idx].detach() + + self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]) + + # test that outputs are equal for slice + self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-2)) + + def check_encoder_decoder_model_standalone(self, config, inputs_dict): + model = NllbMoeModel(config=config).to(torch_device).eval() + outputs = model(**inputs_dict) + + encoder_last_hidden_state = outputs.encoder_last_hidden_state + last_hidden_state = outputs.last_hidden_state + + with tempfile.TemporaryDirectory() as tmpdirname: + encoder = model.get_encoder() + encoder.save_pretrained(tmpdirname) + encoder = NllbMoeEncoder.from_pretrained(tmpdirname).to(torch_device) + + encoder_last_hidden_state_2 = encoder(inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"])[ + 0 + ] + + self.parent.assertTrue((encoder_last_hidden_state_2 - encoder_last_hidden_state).abs().max().item() < 1e-3) + + with tempfile.TemporaryDirectory() as tmpdirname: + decoder = model.get_decoder() + decoder.save_pretrained(tmpdirname) + decoder = NllbMoeDecoder.from_pretrained(tmpdirname).to(torch_device) + + last_hidden_state_2 = decoder( + input_ids=inputs_dict["decoder_input_ids"], + attention_mask=inputs_dict["decoder_attention_mask"], + encoder_hidden_states=encoder_last_hidden_state, + encoder_attention_mask=inputs_dict["attention_mask"], + )[0] + + self.parent.assertTrue((last_hidden_state_2 - last_hidden_state).abs().max().item() < 1e-3) + + +@require_torch +class NllbMoeModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): + all_model_classes = (NllbMoeModel, NllbMoeForConditionalGeneration) if is_torch_available() else () + all_generative_model_classes = (NllbMoeForConditionalGeneration,) if is_torch_available() else () + pipeline_model_mapping = ( + { + "conversational": NllbMoeForConditionalGeneration, + "feature-extraction": NllbMoeModel, + "summarization": NllbMoeForConditionalGeneration, + "text2text-generation": NllbMoeForConditionalGeneration, + } + if is_torch_available() + else {} + ) + is_encoder_decoder = True + fx_compatible = False + test_pruning = False + test_missing_keys = True + test_torchscript = False + + def setUp(self): + self.model_tester = NllbMoeModelTester(self) + self.config_tester = ConfigTester(self, config_class=NllbMoeConfig) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_save_load_strict(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs() + for model_class in self.all_model_classes: + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True) + self.assertEqual(info["missing_keys"], []) + + def test_decoder_model_past_with_large_inputs(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs) + + def test_encoder_decoder_model_standalone(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common() + self.model_tester.check_encoder_decoder_model_standalone(*config_and_inputs) + + def test_inputs_embeds(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in (NllbMoeModel, NllbMoeForConditionalGeneration): + model = model_class(config) + model.to(torch_device) + model.eval() + + inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) + + if not self.is_encoder_decoder: + input_ids = inputs["input_ids"] + del inputs["input_ids"] + else: + encoder_input_ids = inputs["input_ids"] + decoder_input_ids = inputs.get("decoder_input_ids", encoder_input_ids) + del inputs["input_ids"] + inputs.pop("decoder_input_ids", None) + + wte = model.get_input_embeddings() + if not self.is_encoder_decoder: + inputs["inputs_embeds"] = wte(input_ids) + else: + inputs["inputs_embeds"] = wte(encoder_input_ids) + inputs["decoder_inputs_embeds"] = wte(decoder_input_ids) + + with torch.no_grad(): + model(**inputs)[0] + + def test_generate_fp16(self): + config, input_dict = self.model_tester.prepare_config_and_inputs() + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + model = NllbMoeForConditionalGeneration(config).eval().to(torch_device) + if torch_device == "cuda": + model.half() + model.generate(input_ids, attention_mask=attention_mask) + model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) + + +@require_torch +@require_sentencepiece +@require_tokenizers +@slow +class NllbMoeModelIntegrationTests(unittest.TestCase): + @require_torch + @cached_property + def model_inputs(self): + return { + "input_ids": torch.LongTensor( + [ + [28768, 248, 6399, 9, 65972, 452, 1925, 629, 123543, 248075, 2, 256047], + [117, 7027, 7195, 202, 44778, 248075, 2, 256047, 1, 1, 1, 1], + ] + ), + "attention_mask": torch.Tensor( + [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]] + ), + "decoder_input_ids": torch.LongTensor([[2, 256057], [2, 256057]]), + } + + @cached_property + def tokenizer(self): + return NllbTokenizer.from_pretrained("ArthurZ/random-nllb-moe-2-experts") + + @cached_property + def big_model(self): + return NllbMoeForConditionalGeneration.from_pretrained("facebook/nllb-moe-54b") + + def inference_no_head(self): + model = NllbMoeModel.from_pretrained("ArthurZ/random-nllb-moe-2-experts").eval() + with torch.no_grad(): + output = model(**self.model_inputs) + # fmt: off + EXPECTED_ENCODER_STATE = torch.Tensor([ 0.3920, -0.1974, -0.0279, 0.3463, -0.8306, -1.0629, -0.4643, 2.0563, 1.1123, 0.3566, -0.9291, -0.3840, -0.2527, -0.9858, 1.5185, -1.1346, 0.0323, -0.9103, -0.3647, -0.4462, -0.9720, -0.3541, 0.1777, -0.4647, 1.6970, -0.9062, 0.2727, -1.0737, 0.8785, 0.4324]) + EXPECTED_DECODER_STATE = torch.Tensor([-6.0425e-02, -2.0015e-01, 6.0575e-02, -8.6366e-01, -1.1310e+00, 6.8369e-01, 7.5615e-01, 7.3555e-01, 2.3071e-01, 1.5954e+00, -7.0728e-01, -2.2647e-01, -1.3292e+00, 4.8246e-01, -6.9153e-01, -1.8199e-02, -7.3664e-01, 1.5902e-03, 1.0760e-01, 1.0298e-01, -9.3933e-01, -4.6567e-01, 8.0417e-01, 1.5243e+00, 5.5844e-01, -9.9239e-02, 1.4885e+00, 7.1527e-02, -5.2612e-01, 9.4435e-02]) + # fmt: on + + torch.testing.assert_allclose( + output.encoder_last_hidden_state[1, 0, :30], EXPECTED_ENCODER_STATE, rtol=6e-3, atol=9e-3 + ) + torch.testing.assert_allclose( + output.last_hidden_state[1, 0, :30], EXPECTED_DECODER_STATE, rtol=6e-3, atol=9e-3 + ) + + def test_inference_logits(self): + r""" + Logits testing to check implementation consistency between `fairseq` implementation + and `transformers` implementation of NLLB-MoE transformers. We only check the logits + of the second sample of the batch, as it is padded. + """ + model = NllbMoeForConditionalGeneration.from_pretrained("ArthurZ/random-nllb-moe-2-experts").eval() + with torch.no_grad(): + output = model(**self.model_inputs) + + # fmt: off + EXPECTED_LOGTIS = torch.Tensor([-0.3059, 0.0000, 9.3029, 0.6456, -0.9148, 1.7836, 0.6478, 0.9438, -0.5272, -0.6617, -1.2717, 0.4564, 0.1345, -0.2301, -1.0140, 1.1427, -1.5535, 0.1337, 0.2082, -0.8112, -0.3842, -0.3377, 0.1256, 0.6450, -0.0452, 0.0219, 1.4274, -0.4991, -0.2063, -0.4409,]) + # fmt: on + torch.testing.assert_allclose(output.logits[1, 0, :30], EXPECTED_LOGTIS, rtol=6e-3, atol=9e-3) + + @unittest.skip("This requires 300GB of RAM") + def test_large_logits(self): + model = self.big_model + with torch.no_grad(): + output = model(**self.model_inputs) + + # fmt: off + EXPECTED_ENCODER_STATE = torch.Tensor([ 0.1696, -0.0059, 0.0489, 0.0479, -0.4222, -0.2178, -0.1372, -0.0860, -0.4249, -0.0081, -0.1186, 0.6678, 0.0160, 0.4140, 0.1799, 0.0672, -0.4941, 0.0173, -0.0740, 0.0845, -0.2197, 0.4465, 0.2268, -0.1752, -0.0562, 0.1033, -0.0869, -0.5490, 0.0582, 0.2165]) + EXPECTED_DECODER_STATE = torch.Tensor([ 0.0374, -0.1055, -0.1060, -0.1711, -0.0540, -0.1183, -0.0779, 0.0610, -0.0279, -0.0848, 0.0222, 0.0372, -0.0298, -0.0861, -0.0354, -0.0103, 0.0538, -0.0148, -0.0105, 0.0224, 0.0629, -0.0291, -0.0671, 0.0173, -0.0066, -0.0245, -0.0499, 0.0760, -0.0067, 0.0086]) + EXPECTED_LOGTIS = torch.Tensor([ 0.3834, 0.2057, 4.5399, 0.8301, 0.4810, 0.9325, 0.9928, 0.9574, 0.5517, 0.9156, 0.2698, 0.6728, 0.7121, 0.3080, 0.4693, 0.5756, 1.0407, 0.2219, 0.3714, 0.5699, 0.5547, 0.8472, 0.3178, 0.1286, 0.1791, 0.9391, 0.5153, -0.2146, 0.1689, 0.6816]) + # fmt: on + + torch.testing.assert_allclose( + output.encoder_last_hidden_state[1, 0, :30], EXPECTED_ENCODER_STATE, rtol=6e-3, atol=9e-3 + ) + torch.testing.assert_allclose( + output.last_hidden_state[1, 0, :30], EXPECTED_DECODER_STATE, rtol=6e-3, atol=9e-3 + ) + torch.testing.assert_allclose(output.logits[1, 0, :30], EXPECTED_LOGTIS, rtol=6e-3, atol=9e-3) + + @unittest.skip("This requires 300GB of RAM") + def test_seq_to_seq_generation(self): + model = self.big_model + tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-moe-54b") + + # first 6 samples of load_dataset("facebook/flores", "eng_Latn-fra_Latn"), devtest. Truth are very similar to the fairseq translation files + FIRST_6_FLORES_200 = [ + 'We now have 4-month-old mice that are non-diabetic that used to be diabetic," he added.', + "Dr. Ehud Ur, professor of medicine at Dalhousie University in Halifax, Nova Scotia and chair of the clinical and scientific division of the Canadian Diabetes Association cautioned that the research is still in its early days.", + "Like some other experts, he is skeptical about whether diabetes can be cured, noting that these findings have no relevance to people who already have Type 1 diabetes.", + "On Monday, Sara Danius, permanent secretary of the Nobel Committee for Literature at the Swedish Academy, publicly announced during a radio program on Sveriges Radio in Sweden the committee, unable to reach Bob Dylan directly about winning the 2016 Nobel Prize in Literature, had abandoned its efforts to reach him.", + 'Danius said, "Right now we are doing nothing. I have called and sent emails to his closest collaborator and received very friendly replies. For now, that is certainly enough."', + "Previously, Ring's CEO, Jamie Siminoff, remarked the company started when his doorbell wasn't audible from his shop in his garage.", + ] + inputs = tokenizer(FIRST_6_FLORES_200, padding=True, return_tensors="pt").to(torch_device) + batch_translation = model.generate(**inputs, forced_bos_token_id=tokenizer.lang_code_to_id["fra_Latn"]) + + EXPECTED_FAIRSEQ_TRANSLATION = [ + '"Nous avons maintenant des souris de 4 mois non diabétiques qui étaient diabétiques", a-t-il ajouté.', + "Le docteur Ehud Ur, professeur de médecine à l'université Dalhousie, à Halifax, en Nouvelle-Écosse, et président de la division clinique et scientifique de l'Association canadienne du diabète, prévient que la recherche n'en est qu'à ses débuts.", + "Comme d'autres spécialistes, il est sceptique quant à la guérison du diabète.", + "Lundi, Sara Danius, secrétaire permanente du Comité Nobel de littérature à l'Académie suédoise, a annoncé publiquement lors d'une émission de radio sur Sveriges Radio en Suède que le comité, incapable de joindre Bob Dylan directement pour lui annoncer le prix Nobel de littérature 2016, avait abandonné ses efforts pour le joindre.", + "Danius a déclaré: \"Pour l'instant, nous ne faisons rien. J'ai appelé et envoyé des courriels à son plus proche collaborateur et j'ai reçu des réponses très amicales. Pour l'instant, c'est certainement suffisant\".", + "Auparavant, le PDG de Ring, Jamie Siminoff, a fait remarquer que la société avait commencé lorsque sa sonnette n'était pas audible depuis son magasin dans son garage.", + ] + + translation = tokenizer.batch_decode( + batch_translation.tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True + ) + assert translation == EXPECTED_FAIRSEQ_TRANSLATION + + +@require_torch +class NllbMoeRouterTest(unittest.TestCase): + r""" + Switch Transformers has different blocks from classic transformer based models. + The Swift MLP contains a Router class, that has to be tested to check if it is correctly implemented + + Original implementation of the routers here: + + """ + config = NllbMoeConfig( + num_experts=4, + hidden_size=32, + d_ff=16, + expert_capacity=4, + ) + batch_size = 2 + sequence_length = 20 + + def test_top_2_routing(self): + # test routing with minimal reproduction + mask = torch.ones((self.batch_size, self.sequence_length), dtype=torch.bool) + mask[0][0] = False + mask[1][0] = False + mask = mask.reshape(-1) + set_seed(0) + hidden_states = torch.rand((self.batch_size, self.sequence_length, self.config.hidden_size)) + classfier = torch.nn.Linear(self.config.hidden_size, self.config.num_experts) + hf_router = NllbMoeTop2Router(self.config) + + _, _, hidden_dim = hidden_states.shape + logits = classfier(hidden_states.reshape((self.batch_size * self.sequence_length), hidden_dim)) + top_1_mask, router_probs = hf_router.route_tokens(logits, padding_mask=mask) + torch.argmax(top_1_mask, dim=-1) + router_mask = router_probs.bool() + set_seed(0) + experts = [ + torch.nn.Linear(hidden_dim, hidden_dim), + torch.nn.Linear(hidden_dim, hidden_dim), + torch.nn.Linear(hidden_dim, hidden_dim), + torch.nn.Linear(hidden_dim, hidden_dim), + ] + hidden_states = hidden_states.reshape((self.batch_size * self.sequence_length), hidden_dim) + masked_hidden_states = torch.einsum("bm,be->ebm", hidden_states, router_mask) + for idx, expert in enumerate(experts): + token_indices = router_mask[:, idx] + combining_weights = router_probs[token_indices, idx] + expert_output = expert(masked_hidden_states[idx, token_indices]) + expert_output *= 1 - self.config.moe_token_dropout + masked_hidden_states[idx, token_indices] = torch.einsum("b,be->be", combining_weights, expert_output) + hidden_states = masked_hidden_states.sum(dim=0).reshape(self.batch_size, self.sequence_length, hidden_dim) + + # fmt: off + EXPECTED_MEAN_FAIRSEQ_HIDDEN_STATES = torch.Tensor([[ 7.0340e-04, 2.7997e-03, -1.3351e-02, -7.6705e-03, -3.5089e-03,3.9773e-03, 7.4593e-03, 1.2566e-02, 3.5860e-03, -2.7448e-02,-1.3731e-02, -1.0534e-02, -1.3606e-02, -1.5048e-02, -2.8914e-03,-5.0371e-03, -1.3963e-03, 6.0076e-03, -1.1380e-02, -1.4620e-02, 5.2401e-03, 8.4660e-04, -1.5319e-03, -1.6735e-02, 1.1302e-02, 3.6119e-03, 4.6084e-03, -1.3458e-02, 7.7792e-05, 1.4312e-02, 4.9107e-03, -5.0936e-03], [-4.4538e-03, 3.1026e-03, 1.4121e-04, -4.8121e-03, -5.6279e-03, 7.2493e-03, 3.9769e-03, 1.1114e-02, -1.5666e-03, -2.3477e-02, 8.7268e-03, 1.3446e-02, -2.8845e-05, -1.7287e-02, 8.7619e-03, -4.5316e-03, -1.2164e-02, 5.7461e-03, -4.5861e-03, -9.3907e-03, 2.9808e-02, 8.9206e-04, -7.6232e-04, -1.4173e-02, 3.0208e-03, 1.5310e-02, 9.7717e-03, 3.1014e-03, 7.8042e-03, 8.0197e-03, 3.4784e-03, -7.1728e-03]]) + # fmt: on + self.assertTrue(torch.allclose(hidden_states.mean(1), EXPECTED_MEAN_FAIRSEQ_HIDDEN_STATES, 1e-4)) + + def test_batch_prioritized_routing(self): + set_seed(0) + config = NllbMoeConfig( + num_experts=4, hidden_size=32, d_ff=16, expert_capacity=4, second_expert_policy="random" + ) + mask = torch.zeros((self.batch_size * self.sequence_length), dtype=torch.bool) + logits = torch.rand((self.batch_size * self.sequence_length, 4)) + config.batch_prioritized_routing = True + router = NllbMoeTop2Router(config) + top_1_mask, _ = router.route_tokens(logits, padding_mask=mask) + # check that the routing is batch first. One of the last token is routed while expert capacity is very small + # this means that it had a greater probability of being routed + assert top_1_mask[-1, 0] == 1 + + def test_second_expert_policy(self): + config = NllbMoeConfig( + num_experts=4, + hidden_size=32, + d_ff=16, + expert_capacity=40, + ) + set_seed(0) + mask = torch.zeros((self.batch_size * self.sequence_length), dtype=torch.bool) + logits = torch.rand((self.batch_size * self.sequence_length, 4)) + + set_seed(0) + config.second_expert_policy = "random" + router = NllbMoeTop2Router(config) + top_1_mask, router_probs = router.route_tokens(logits, padding_mask=mask) + + set_seed(0) + config.second_expert_policy = "sampling" + router = NllbMoeTop2Router(config) + top_1_mask_sp, router_probs_sp = router.route_tokens(logits, padding_mask=mask) + + set_seed(0) + config.second_expert_policy = "all" + router = NllbMoeTop2Router(config) + top_1_mask_all, router_probs_all = router.route_tokens(logits, padding_mask=mask) + + # fmt: off + EXPECTED_ROUTER_ALL = torch.tensor([[0.3902, 0.0000, 0.0000, 0.6098], [0.0000, 0.0000, 0.7770, 0.2230], [0.0000, 0.0000, 0.2726, 0.7274], [0.4221, 0.0000, 0.5779, 0.0000], [0.0000, 0.0000, 0.7810, 0.2190], [0.5518, 0.4482, 0.0000, 0.0000], [0.0000, 0.4060, 0.5940, 0.0000], [0.7340, 0.0000, 0.0000, 0.2660], [0.4778, 0.5222, 0.0000, 0.0000], [0.0000, 0.3984, 0.0000, 0.6016], [0.0000, 0.0548, 0.9452, 0.0000], [0.6796, 0.0000, 0.0000, 0.3204], [0.0700, 0.0000, 0.9300, 0.0000], [0.1854, 0.0000, 0.8146, 0.0000], [0.6775, 0.3225, 0.0000, 0.0000], [0.0000, 0.0000, 0.5027, 0.4973], [0.0000, 0.6577, 0.0000, 0.3423], [0.0000, 0.7767, 0.0000, 0.2233], [0.1944, 0.8056, 0.0000, 0.0000], [0.0000, 0.3073, 0.0000, 0.6927], [0.0000, 0.5655, 0.4345, 0.0000], [0.5791, 0.0000, 0.0000, 0.4209], [0.0440, 0.0000, 0.9560, 0.0000], [0.0083, 0.9917, 0.0000, 0.0000], [0.0000, 0.8395, 0.0000, 0.1605], [0.0000, 0.1458, 0.0000, 0.8542], [0.0000, 0.8534, 0.1466, 0.0000], [0.4938, 0.0000, 0.0000, 0.5062], [0.1329, 0.8671, 0.0000, 0.0000], [0.3058, 0.0000, 0.6942, 0.0000], [0.4458, 0.0000, 0.0000, 0.5542], [0.9053, 0.0947, 0.0000, 0.0000], [0.0000, 0.7563, 0.2437, 0.0000], [0.0000, 0.0000, 0.4096, 0.5904], [0.4551, 0.0000, 0.0000, 0.5449], [0.8502, 0.1498, 0.0000, 0.0000], [0.0000, 0.6312, 0.3688, 0.0000], [0.8920, 0.0000, 0.0000, 0.1080], [0.1913, 0.0000, 0.0000, 0.8087], [0.2491, 0.7509, 0.0000, 0.0000]]) + EXPECTED_ROUTER_SP = torch.tensor([[0.0000, 0.6539, 0.0000, 0.3461], [0.0000, 0.0000, 0.3998, 0.6002], [0.0000, 0.5574, 0.0000, 0.4426], [0.0000, 0.0000, 0.4441, 0.5559], [0.0000, 0.6545, 0.3455, 0.0000], [0.4419, 0.5581, 0.0000, 0.0000], [0.0000, 0.4014, 0.5986, 0.0000], [0.3215, 0.0000, 0.0000, 0.6785], [0.4765, 0.5235, 0.0000, 0.0000], [0.0000, 0.5467, 0.0000, 0.4533], [0.0000, 0.4156, 0.5844, 0.0000], [0.3370, 0.0000, 0.6630, 0.0000], [0.0000, 0.0000, 0.4558, 0.5442], [0.4659, 0.0000, 0.5341, 0.0000], [0.6179, 0.3821, 0.0000, 0.0000], [0.6277, 0.0000, 0.3723, 0.0000], [0.5836, 0.4164, 0.0000, 0.0000], [0.0000, 0.6600, 0.0000, 0.3400], [0.0000, 0.4933, 0.0000, 0.5067], [0.6016, 0.0000, 0.0000, 0.3984], [0.0000, 0.5160, 0.4840, 0.0000], [0.5799, 0.0000, 0.0000, 0.4201], [0.0000, 0.0000, 0.4826, 0.5174], [0.5426, 0.4574, 0.0000, 0.0000], [0.5362, 0.4638, 0.0000, 0.0000], [0.6448, 0.0000, 0.0000, 0.3552], [0.0000, 0.5909, 0.4091, 0.0000], [0.4196, 0.0000, 0.0000, 0.5804], [0.3191, 0.6809, 0.0000, 0.0000], [0.0000, 0.0000, 0.4886, 0.5114], [0.4899, 0.0000, 0.0000, 0.5101], [0.4123, 0.0000, 0.5877, 0.0000], [0.0000, 0.3736, 0.0000, 0.6264], [0.0000, 0.0000, 0.6009, 0.3991], [0.4246, 0.0000, 0.0000, 0.5754], [0.4997, 0.0000, 0.5003, 0.0000], [0.0000, 0.3595, 0.6405, 0.0000], [0.5433, 0.0000, 0.0000, 0.4567], [0.0000, 0.6806, 0.0000, 0.3194], [0.6689, 0.3311, 0.0000, 0.0000]]) + EXPECTED_ROUTER = torch.tensor([[0.4324, 0.5676, 0.0000, 0.0000], [0.0000, 0.4348, 0.0000, 0.5652], [0.4559, 0.5441, 0.0000, 0.0000], [0.0000, 0.0000, 0.0000, 1.0000], [0.4744, 0.5256, 0.0000, 0.0000], [0.0000, 0.5103, 0.0000, 0.4897], [0.0000, 0.0000, 1.0000, 0.0000], [0.0000, 0.0000, 0.0000, 1.0000], [0.0000, 1.0000, 0.0000, 0.0000], [0.0000, 0.5467, 0.0000, 0.4533], [0.0000, 0.0000, 1.0000, 0.0000], [0.0000, 0.0000, 1.0000, 0.0000], [0.0000, 0.0000, 0.0000, 1.0000], [0.0000, 0.0000, 1.0000, 0.0000], [1.0000, 0.0000, 0.0000, 0.0000], [0.5063, 0.4937, 0.0000, 0.0000], [0.5396, 0.0000, 0.0000, 0.4604], [0.4576, 0.5424, 0.0000, 0.0000], [0.0000, 0.0000, 0.0000, 1.0000], [0.5134, 0.0000, 0.4866, 0.0000], [0.0000, 0.5160, 0.4840, 0.0000], [0.5439, 0.0000, 0.4561, 0.0000], [0.4849, 0.0000, 0.0000, 0.5151], [0.5426, 0.4574, 0.0000, 0.0000], [0.5362, 0.4638, 0.0000, 0.0000], [1.0000, 0.0000, 0.0000, 0.0000], [0.0000, 1.0000, 0.0000, 0.0000], [0.0000, 0.4448, 0.0000, 0.5552], [0.0000, 1.0000, 0.0000, 0.0000], [0.0000, 0.0000, 0.4886, 0.5114], [0.4899, 0.0000, 0.0000, 0.5101], [0.0000, 0.0000, 0.5296, 0.4704], [0.0000, 0.0000, 0.4469, 0.5531], [0.0000, 0.4053, 0.5947, 0.0000], [0.0000, 0.0000, 0.4460, 0.5540], [0.4997, 0.0000, 0.5003, 0.0000], [0.0000, 0.0000, 0.5851, 0.4149], [1.0000, 0.0000, 0.0000, 0.0000], [0.0000, 0.5010, 0.4990, 0.0000], [1.0000, 0.0000, 0.0000, 0.0000]]) + + EXPECTED_TOP_1_ALL = torch.LongTensor([[0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [0, 0, 0, 1], [1, 0, 0, 0], [0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0]]) + EXPECTED_TOP_1_SP = torch.LongTensor([[0, 1, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], [0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 0, 1], [0, 0, 1, 0], [1, 0, 0, 0], [1, 0, 0, 0], [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [1, 0, 0, 0], [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 1, 0, 0], [1, 0, 0, 0]]) + # `sampling` and `random` do not affect the mask of the top_1 router + # fmt: on + + torch.testing.assert_allclose(router_probs_all, EXPECTED_ROUTER_ALL, 1e-4, 1e-4) + torch.testing.assert_allclose(router_probs_sp, EXPECTED_ROUTER_SP, 1e-4, 1e-4) + torch.testing.assert_allclose(router_probs, EXPECTED_ROUTER, 1e-4, 1e-4) + + torch.testing.assert_allclose(top_1_mask_all, EXPECTED_TOP_1_ALL, 1e-4, 1e-4) + torch.testing.assert_allclose(top_1_mask_sp, EXPECTED_TOP_1_SP, 1e-4, 1e-4) + torch.testing.assert_allclose(top_1_mask, EXPECTED_TOP_1_SP, 1e-4, 1e-4) diff --git a/utils/check_repo.py b/utils/check_repo.py index bcdecf729e4a..8a0c205ad0b8 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -57,6 +57,8 @@ # Being in this list is an exception and should **not** be the rule. IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [ # models to ignore for not tested + "NllbMoeDecoder", + "NllbMoeEncoder", "LlamaDecoder", # Building part of bigger (tested) model. "Blip2QFormerModel", # Building part of bigger (tested) model. "DetaEncoder", # Building part of bigger (tested) model. From 7ea98760f130efe9a4c0c7854915d145023aca70 Mon Sep 17 00:00:00 2001 From: Kshiteej K Date: Tue, 28 Mar 2023 00:30:16 +0530 Subject: [PATCH 209/277] [neptune] fix checkpoint bug with relative out_dir (#22102) * [neptune] fix checkpoint bug with relative out_dir * update imports * reformat with black * check neptune without imports * fix typing-related issue * run black on code * use os.path.sep instead of raw \ * simplify imports and remove type annotation * make ruff happy * apply review suggestions --------- Co-authored-by: Aleksander Wojnarowicz --- src/transformers/integrations.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/transformers/integrations.py b/src/transformers/integrations.py index 14857f830831..52e4d92148ec 100644 --- a/src/transformers/integrations.py +++ b/src/transformers/integrations.py @@ -31,6 +31,7 @@ from . import __version__ as version from .utils import flatten_dict, is_datasets_available, is_torch_available, logging +from .utils.versions import importlib_metadata logger = logging.get_logger(__name__) @@ -53,9 +54,19 @@ except (ImportError, ValueError): _has_comet = False -_has_neptune = importlib.util.find_spec("neptune") is not None +_has_neptune = ( + importlib.util.find_spec("neptune") is not None or importlib.util.find_spec("neptune-client") is not None +) if TYPE_CHECKING and _has_neptune: - from neptune.new.metadata_containers.run import Run + try: + _neptune_version = importlib_metadata.version("neptune") + logger.info(f"Neptune version {_neptune_version} available.") + except importlib_metadata.PackageNotFoundError: + try: + _neptune_version = importlib_metadata.version("neptune-client") + logger.info(f"Neptune-client version {_neptune_version} available.") + except importlib_metadata.PackageNotFoundError: + _has_neptune = False from .trainer_callback import ProgressCallback, TrainerCallback # noqa: E402 from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun, IntervalStrategy # noqa: E402 @@ -1155,7 +1166,7 @@ def __init__( project: Optional[str] = None, name: Optional[str] = None, base_namespace: str = "finetuning", - run: Optional["Run"] = None, + run=None, log_parameters: bool = True, log_checkpoints: Optional[str] = None, **neptune_run_kwargs, @@ -1163,15 +1174,15 @@ def __init__( if not is_neptune_available(): raise ValueError( "NeptuneCallback requires the Neptune client library to be installed. " - "To install the library, run `pip install neptune-client`." + "To install the library, run `pip install neptune`." ) - from neptune.new.metadata_containers.run import Run - try: - from neptune.new.integrations.utils import verify_type + from neptune import Run + from neptune.internal.utils import verify_type except ImportError: from neptune.new.internal.utils import verify_type + from neptune.new.metadata_containers.run import Run verify_type("api_token", api_token, (str, type(None))) verify_type("project", project, (str, type(None))) @@ -1288,7 +1299,10 @@ def _log_model_checkpoint(self, source_directory: str, checkpoint: str): if self._volatile_checkpoints_dir is not None: consistent_checkpoint_path = os.path.join(self._volatile_checkpoints_dir, checkpoint) try: - shutil.copytree(relative_path, os.path.join(consistent_checkpoint_path, relative_path)) + # Remove leading ../ from a relative path. + cpkt_path = relative_path.replace("..", "").lstrip(os.path.sep) + copy_path = os.path.join(consistent_checkpoint_path, cpkt_path) + shutil.copytree(relative_path, copy_path) target_path = consistent_checkpoint_path except IOError as e: logger.warning( From 3d30610ff289d731d6dc02cb6db5a1d4162c1ca8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 20:23:55 -0400 Subject: [PATCH 210/277] Bump redis from 4.1.4 to 4.5.3 in /examples/research_projects/decision_transformer (#22410) Bump redis in /examples/research_projects/decision_transformer Bumps [redis](https://github.com/redis/redis-py) from 4.1.4 to 4.5.3. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v4.1.4...v4.5.3) --- updated-dependencies: - dependency-name: redis dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../research_projects/decision_transformer/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/research_projects/decision_transformer/requirements.txt b/examples/research_projects/decision_transformer/requirements.txt index 112141e172dd..b8545669e2cf 100644 --- a/examples/research_projects/decision_transformer/requirements.txt +++ b/examples/research_projects/decision_transformer/requirements.txt @@ -175,7 +175,7 @@ pytz==2022.1 pytz-deprecation-shim==0.1.0.post0 PyYAML==6.0 ray==1.11.0 -redis==4.1.4 +redis==4.5.3 regex==2022.3.15 requests==2.27.1 requests-oauthlib==1.3.1 From f7bc749f1eea67c61e4349594f9461d0dedcf6fe Mon Sep 17 00:00:00 2001 From: fpgaminer Date: Tue, 28 Mar 2023 06:09:17 -0700 Subject: [PATCH 211/277] Fix bug in perplexity guide calculations and update perplexity numbers. Fixes #22348 (#22411) Fix bug in perplexity guide calculations and update perplexity numbers. --- docs/source/en/perplexity.mdx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/source/en/perplexity.mdx b/docs/source/en/perplexity.mdx index 01f861c99c5e..76ffd3385682 100644 --- a/docs/source/en/perplexity.mdx +++ b/docs/source/en/perplexity.mdx @@ -115,11 +115,10 @@ for begin_loc in tqdm(range(0, seq_len, stride)): with torch.no_grad(): outputs = model(input_ids, labels=target_ids) - # loss is calculated using CrossEntropyLoss which averages over input tokens. - # Multiply it with trg_len to get the summation instead of average. - # We will take average over all the tokens to get the true average - # in the last step of this example. - neg_log_likelihood = outputs.loss * trg_len + # loss is calculated using CrossEntropyLoss which averages over valid labels + # N.B. the model only calculates loss over trg_len - 1 labels, because it internally shifts the labels + # to the left by 1. + neg_log_likelihood = outputs.loss nlls.append(neg_log_likelihood) @@ -127,14 +126,14 @@ for begin_loc in tqdm(range(0, seq_len, stride)): if end_loc == seq_len: break -ppl = torch.exp(torch.stack(nlls).sum() / end_loc) +ppl = torch.exp(torch.stack(nlls).mean()) ``` Running this with the stride length equal to the max input length is equivalent to the suboptimal, non-sliding-window strategy we discussed above. The smaller the stride, the more context the model will have in making each prediction, and the better the reported perplexity will typically be. -When we run the above with `stride = 1024`, i.e. no overlap, the resulting PPL is `19.64`, which is about the same +When we run the above with `stride = 1024`, i.e. no overlap, the resulting PPL is `19.44`, which is about the same as the `19.93` reported in the GPT-2 paper. By using `stride = 512` and thereby employing our striding window -strategy, this jumps down to `16.44`. This is not only a more favorable score, but is calculated in a way that is +strategy, this jumps down to `16.45`. This is not only a more favorable score, but is calculated in a way that is closer to the true autoregressive decomposition of a sequence likelihood. From c426a6d4678b88cb6cb09b589dedee5a52b11e62 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Tue, 28 Mar 2023 06:17:03 -0700 Subject: [PATCH 212/277] [performance] ensure `causal_mask` is created directly on device (#22378) * ensure causal_mask is created directly on device * add copy tag to opt, update bart implementation * add device to all _make_causal_mask copies * formatting fixes * more manual fixes due to unlinked versions of _prepare_decoder_attention_mask --- src/transformers/models/bart/modeling_bart.py | 17 +++++++++++------ .../modeling_bigbird_pegasus.py | 18 ++++++++++++------ .../models/biogpt/modeling_biogpt.py | 17 +++++++++++------ .../models/blenderbot/modeling_blenderbot.py | 17 +++++++++++------ .../modeling_blenderbot_small.py | 17 +++++++++++------ .../models/informer/modeling_informer.py | 17 +++++++++++------ .../models/llama/modeling_llama.py | 17 +++++++++++------ .../models/m2m_100/modeling_m2m_100.py | 17 +++++++++++------ .../models/marian/modeling_marian.py | 17 +++++++++++------ .../models/mbart/modeling_mbart.py | 17 +++++++++++------ src/transformers/models/mvp/modeling_mvp.py | 17 +++++++++++------ .../models/nllb_moe/modeling_nllb_moe.py | 17 +++++++++++------ src/transformers/models/opt/modeling_opt.py | 18 ++++++++++++------ .../models/pegasus/modeling_pegasus.py | 17 +++++++++++------ .../models/pegasus_x/modeling_pegasus_x.py | 17 +++++++++++------ .../models/plbart/modeling_plbart.py | 17 +++++++++++------ .../speech_to_text/modeling_speech_to_text.py | 17 +++++++++++------ .../modeling_speech_to_text_2.py | 17 +++++++++++------ .../models/speecht5/modeling_speecht5.py | 17 +++++++++++------ .../modeling_time_series_transformer.py | 17 +++++++++++------ .../models/trocr/modeling_trocr.py | 17 +++++++++++------ .../models/whisper/modeling_whisper.py | 17 +++++++++++------ src/transformers/models/xglm/modeling_xglm.py | 17 +++++++++++------ 23 files changed, 255 insertions(+), 138 deletions(-) diff --git a/src/transformers/models/bart/modeling_bart.py b/src/transformers/models/bart/modeling_bart.py index 3706c6c0fe06..8bfa2ee740f5 100755 --- a/src/transformers/models/bart/modeling_bart.py +++ b/src/transformers/models/bart/modeling_bart.py @@ -87,18 +87,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start return shifted_input_ids -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -920,8 +922,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py index ad22199e66e6..6c6bfd431528 100755 --- a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py +++ b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py @@ -78,18 +78,21 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start return shifted_input_ids -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -2101,8 +2104,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/biogpt/modeling_biogpt.py b/src/transformers/models/biogpt/modeling_biogpt.py index 5096336e39bc..178e67f8e1bc 100755 --- a/src/transformers/models/biogpt/modeling_biogpt.py +++ b/src/transformers/models/biogpt/modeling_biogpt.py @@ -43,18 +43,20 @@ # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -470,8 +472,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/blenderbot/modeling_blenderbot.py b/src/transformers/models/blenderbot/modeling_blenderbot.py index b65a74983685..dad8b7e38929 100755 --- a/src/transformers/models/blenderbot/modeling_blenderbot.py +++ b/src/transformers/models/blenderbot/modeling_blenderbot.py @@ -77,18 +77,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -855,8 +857,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py index 69e1b069e77a..aae00db06917 100755 --- a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py +++ b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py @@ -73,18 +73,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -850,8 +852,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/informer/modeling_informer.py b/src/transformers/models/informer/modeling_informer.py index 10524c70bc42..1d0451add50d 100644 --- a/src/transformers/models/informer/modeling_informer.py +++ b/src/transformers/models/informer/modeling_informer.py @@ -230,18 +230,20 @@ def nll(input: torch.distributions.Distribution, target: torch.Tensor) -> torch. # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -1281,8 +1283,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index cc9e01e4aa5a..6d79536627fc 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -39,18 +39,20 @@ # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -481,8 +483,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/m2m_100/modeling_m2m_100.py b/src/transformers/models/m2m_100/modeling_m2m_100.py index 4ed8d2e5c167..f02cd209de43 100755 --- a/src/transformers/models/m2m_100/modeling_m2m_100.py +++ b/src/transformers/models/m2m_100/modeling_m2m_100.py @@ -73,18 +73,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -1002,8 +1004,11 @@ def forward( combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None and combined_attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/marian/modeling_marian.py b/src/transformers/models/marian/modeling_marian.py index c2befbc1819a..23075a055f6c 100755 --- a/src/transformers/models/marian/modeling_marian.py +++ b/src/transformers/models/marian/modeling_marian.py @@ -75,18 +75,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -861,8 +863,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/mbart/modeling_mbart.py b/src/transformers/models/mbart/modeling_mbart.py index 0bf3fb62f769..fe12c82dcb7b 100755 --- a/src/transformers/models/mbart/modeling_mbart.py +++ b/src/transformers/models/mbart/modeling_mbart.py @@ -80,18 +80,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int): # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -907,8 +909,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/mvp/modeling_mvp.py b/src/transformers/models/mvp/modeling_mvp.py index 0d787eda2980..6b2bdcb93d5a 100644 --- a/src/transformers/models/mvp/modeling_mvp.py +++ b/src/transformers/models/mvp/modeling_mvp.py @@ -91,18 +91,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -1044,8 +1046,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/nllb_moe/modeling_nllb_moe.py b/src/transformers/models/nllb_moe/modeling_nllb_moe.py index 5bfe44e900f1..0d2572d9d4e7 100644 --- a/src/transformers/models/nllb_moe/modeling_nllb_moe.py +++ b/src/transformers/models/nllb_moe/modeling_nllb_moe.py @@ -78,18 +78,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -1349,8 +1351,11 @@ def forward( combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None and combined_attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 110b81721657..6ecedc2173e7 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -64,18 +64,21 @@ ] -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -533,8 +536,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/pegasus/modeling_pegasus.py b/src/transformers/models/pegasus/modeling_pegasus.py index e39ed97daa17..bfbe0b2aba0d 100755 --- a/src/transformers/models/pegasus/modeling_pegasus.py +++ b/src/transformers/models/pegasus/modeling_pegasus.py @@ -74,18 +74,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -881,8 +883,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/pegasus_x/modeling_pegasus_x.py b/src/transformers/models/pegasus_x/modeling_pegasus_x.py index 44f3818a2478..f5375b5079b3 100755 --- a/src/transformers/models/pegasus_x/modeling_pegasus_x.py +++ b/src/transformers/models/pegasus_x/modeling_pegasus_x.py @@ -92,18 +92,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -1147,8 +1149,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/plbart/modeling_plbart.py b/src/transformers/models/plbart/modeling_plbart.py index 60122c5d0f14..f00cc8caa19d 100644 --- a/src/transformers/models/plbart/modeling_plbart.py +++ b/src/transformers/models/plbart/modeling_plbart.py @@ -79,18 +79,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int): # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -883,8 +885,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/speech_to_text/modeling_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_speech_to_text.py index d08863f8353f..b5037faf4c9f 100755 --- a/src/transformers/models/speech_to_text/modeling_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_speech_to_text.py @@ -64,18 +64,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -893,8 +895,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py b/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py index 319589eab144..cfb86143b881 100755 --- a/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py +++ b/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py @@ -44,18 +44,20 @@ # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -501,8 +503,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/speecht5/modeling_speecht5.py b/src/transformers/models/speecht5/modeling_speecht5.py index 01ac79af472e..c977414a5263 100644 --- a/src/transformers/models/speecht5/modeling_speecht5.py +++ b/src/transformers/models/speecht5/modeling_speecht5.py @@ -82,18 +82,20 @@ def shift_spectrograms_right(input_values: torch.Tensor): # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -1552,8 +1554,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py b/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py index 286315a19615..812a8d6b4d01 100644 --- a/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py +++ b/src/transformers/models/time_series_transformer/modeling_time_series_transformer.py @@ -225,18 +225,20 @@ def weighted_average(input_tensor: torch.Tensor, weights: Optional[torch.Tensor] # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -1007,8 +1009,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/trocr/modeling_trocr.py b/src/transformers/models/trocr/modeling_trocr.py index e6853d0c5a8e..7e2e11c00e22 100644 --- a/src/transformers/models/trocr/modeling_trocr.py +++ b/src/transformers/models/trocr/modeling_trocr.py @@ -44,18 +44,20 @@ # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -524,8 +526,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/whisper/modeling_whisper.py b/src/transformers/models/whisper/modeling_whisper.py index d6ec39168cd1..cdf769871542 100644 --- a/src/transformers/models/whisper/modeling_whisper.py +++ b/src/transformers/models/whisper/modeling_whisper.py @@ -69,18 +69,20 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -917,8 +919,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] diff --git a/src/transformers/models/xglm/modeling_xglm.py b/src/transformers/models/xglm/modeling_xglm.py index 8a19557932f9..64cec28edf2c 100755 --- a/src/transformers/models/xglm/modeling_xglm.py +++ b/src/transformers/models/xglm/modeling_xglm.py @@ -114,18 +114,20 @@ # Copied from transformers.models.bart.modeling_bart._make_causal_mask -def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): """ Make causal mask used for bi-directional self-attention. """ bsz, tgt_len = input_ids_shape - mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min)) - mask_cond = torch.arange(mask.size(-1)) + mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device) + mask_cond = torch.arange(mask.size(-1), device=device) mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) @@ -578,8 +580,11 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em combined_attention_mask = None if input_shape[-1] > 1: combined_attention_mask = _make_causal_mask( - input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length - ).to(inputs_embeds.device) + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] From 075541914db2eb9d14ecaf9da7d034f5f35f50e2 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Tue, 28 Mar 2023 15:42:02 +0100 Subject: [PATCH 213/277] MBart: Fix docs and doctests (#22422) Fix docs and doctests --- .../models/mbart/modeling_mbart.py | 5 +-- .../models/mbart/modeling_tf_mbart.py | 35 +++++++++++-------- utils/documentation_tests.txt | 3 +- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/transformers/models/mbart/modeling_mbart.py b/src/transformers/models/mbart/modeling_mbart.py index fe12c82dcb7b..778d132ac3f0 100755 --- a/src/transformers/models/mbart/modeling_mbart.py +++ b/src/transformers/models/mbart/modeling_mbart.py @@ -561,7 +561,7 @@ def dummy_inputs(self): >>> inputs = tokenizer(example_english_phrase, return_tensors="pt") >>> # Translate - >>> generated_ids = model.generate(inputs["input_ids"], num_beams=4, max_length=5) + >>> generated_ids = model.generate(**inputs, num_beams=4, max_length=5) >>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] '42 este răspuns' ``` @@ -1266,7 +1266,8 @@ def forward( @add_start_docstrings( - "The MBART Model with a language modeling head. Can be used for summarization.", MBART_START_DOCSTRING + "The MBART Model with a language modeling head. Can be used for summarization, after fine-tuning the pretrained models.", + MBART_START_DOCSTRING, ) class MBartForConditionalGeneration(MBartPreTrainedModel): base_model_prefix = "model" diff --git a/src/transformers/models/mbart/modeling_tf_mbart.py b/src/transformers/models/mbart/modeling_tf_mbart.py index 0f6e18b0b386..6f48062fc637 100644 --- a/src/transformers/models/mbart/modeling_tf_mbart.py +++ b/src/transformers/models/mbart/modeling_tf_mbart.py @@ -619,37 +619,44 @@ def serving(self, inputs): """ MBART_GENERATION_EXAMPLE = r""" - Summarization example: + Translation example: ```python - >>> from transformers import AutoTokenizer, TFMBartForConditionalGeneration, MBartConfig + >>> from transformers import AutoTokenizer, TFMBartForConditionalGeneration - >>> model = TFMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") - >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") + >>> model = TFMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-en-ro") + >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-en-ro") - >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen." - >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="tf") + >>> example_english_phrase = "42 is the answer" + >>> inputs = tokenizer(example_english_phrase, return_tensors="tf") - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"], num_beams=4, max_length=5) - >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + >>> # Translate + >>> generated_ids = model.generate(**inputs, num_beams=4, max_length=5) + >>> tokenizer.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + '42 este răspuns' ``` Mask filling example: ```python >>> from transformers import AutoTokenizer, TFMBartForConditionalGeneration + >>> import tensorflow as tf - >>> model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") + >>> model = TFMBartForConditionalGeneration.from_pretrained("facebook/mbart-large-cc25") >>> tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25") >>> # de_DE is the language symbol id for German >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE" - >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors="tf")["input_ids"] + >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors="tf")["input_ids"] >>> logits = model(input_ids).logits - >>> probs = tf.nn.softmax(logits[0]) - >>> # probs[5] is associated with the mask token + + >>> masked_index = tf.where(input_ids[0] == tokenizer.mask_token_id)[0, 0] + >>> probs = tf.nn.softmax(logits[0, masked_index], axis=0) + >>> values, predictions = tf.math.top_k(probs, 5) + + >>> tokenizer.decode(predictions).split() + ['nett', 'sehr', 'ganz', 'nicht', 'so'] ``` """ @@ -1299,7 +1306,7 @@ def call(self, x): @add_start_docstrings( - "The MBART Model with a language modeling head. Can be used for summarization.", + "The MBART Model with a language modeling head. Can be used for summarization, after fine-tuning the pretrained models.", MBART_START_DOCSTRING, ) class TFMBartForConditionalGeneration(TFMBartPreTrainedModel, TFCausalLanguageModelingLoss): diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 3357c1d56952..ddb85203948c 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -120,6 +120,7 @@ src/transformers/models/maskformer/configuration_maskformer.py src/transformers/models/maskformer/modeling_maskformer.py src/transformers/models/mbart/configuration_mbart.py src/transformers/models/mbart/modeling_mbart.py +src/transformers/models/mbart/modeling_tf_mbart.py src/transformers/models/mctct/configuration_mctct.py src/transformers/models/megatron_bert/configuration_megatron_bert.py src/transformers/models/mobilebert/configuration_mobilebert.py @@ -479,4 +480,4 @@ src/transformers/models/m2m_100/tokenization_m2m_100.py src/transformers/models/marian/tokenization_marian.py src/transformers/models/roformer/tokenization_roformer.py src/transformers/models/roformer/tokenization_roformer_fast.py -src/transformers/models/transfo_xl/tokenization_transfo_xl.py \ No newline at end of file +src/transformers/models/transfo_xl/tokenization_transfo_xl.py From dc38b93c48b8522e4eb5dfc8233c87e3f4e9bf59 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Wed, 29 Mar 2023 13:21:07 +0200 Subject: [PATCH 214/277] Add clean_up_tokenization_spaces to config (#22341) * add draft changes * fix failing wav2vec * style * make sure that the argument is saved + add tests * style * fixup * update test * default clean_up_tokenization_spaces to False for Bloom and Llama * Update code based on review Co-authored-by: Nicolas Patry * style * quality --------- Co-authored-by: Nicolas Patry --- .../models/big_bird/tokenization_big_bird.py | 7 ++- .../models/bloom/tokenization_bloom_fast.py | 2 + .../models/codegen/tokenization_codegen.py | 7 +-- .../codegen/tokenization_codegen_fast.py | 7 +-- .../models/fnet/tokenization_fnet.py | 7 ++- .../models/llama/tokenization_llama.py | 9 +++- .../models/marian/tokenization_marian.py | 10 +++-- .../models/wav2vec2/tokenization_wav2vec2.py | 22 ++++++--- .../tokenization_wav2vec2_phoneme.py | 15 ++++--- .../models/whisper/tokenization_whisper.py | 7 +-- .../whisper/tokenization_whisper_fast.py | 7 +-- .../models/xlnet/tokenization_xlnet.py | 7 ++- src/transformers/tokenization_utils.py | 7 ++- src/transformers/tokenization_utils_base.py | 26 +++++++---- src/transformers/tokenization_utils_fast.py | 7 ++- tests/test_tokenization_common.py | 45 +++++++++++++++++++ 16 files changed, 150 insertions(+), 42 deletions(-) diff --git a/src/transformers/models/big_bird/tokenization_big_bird.py b/src/transformers/models/big_bird/tokenization_big_bird.py index bd6f90ef027a..5220366df4d2 100644 --- a/src/transformers/models/big_bird/tokenization_big_bird.py +++ b/src/transformers/models/big_bird/tokenization_big_bird.py @@ -204,7 +204,7 @@ def _decode( self, token_ids: List[int], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, spaces_between_special_tokens: bool = True, **kwargs, ) -> str: @@ -237,6 +237,11 @@ def _decode( else: text = "".join(sub_texts) + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) return clean_text diff --git a/src/transformers/models/bloom/tokenization_bloom_fast.py b/src/transformers/models/bloom/tokenization_bloom_fast.py index 800c73f0250b..d20065c90338 100644 --- a/src/transformers/models/bloom/tokenization_bloom_fast.py +++ b/src/transformers/models/bloom/tokenization_bloom_fast.py @@ -115,6 +115,7 @@ def __init__( eos_token="", pad_token="", add_prefix_space=False, + clean_up_tokenization_spaces=False, **kwargs, ): super().__init__( @@ -126,6 +127,7 @@ def __init__( eos_token=eos_token, pad_token=pad_token, add_prefix_space=add_prefix_space, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs, ) pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()) diff --git a/src/transformers/models/codegen/tokenization_codegen.py b/src/transformers/models/codegen/tokenization_codegen.py index 0c01e63471dc..14d79bb1cebe 100644 --- a/src/transformers/models/codegen/tokenization_codegen.py +++ b/src/transformers/models/codegen/tokenization_codegen.py @@ -320,7 +320,7 @@ def decode( self, token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, truncate_before_pattern: Optional[List[str]] = None, **kwargs, ) -> str: @@ -335,8 +335,9 @@ def decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). truncate_before_pattern (`List[str]`, *optional*, defaults to `None`): A list of regular expression strings that will be used to truncate the returned string. This can be used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning diff --git a/src/transformers/models/codegen/tokenization_codegen_fast.py b/src/transformers/models/codegen/tokenization_codegen_fast.py index 83e5e30d0b0b..fad5e24dbcf5 100644 --- a/src/transformers/models/codegen/tokenization_codegen_fast.py +++ b/src/transformers/models/codegen/tokenization_codegen_fast.py @@ -187,7 +187,7 @@ def decode( self, token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, truncate_before_pattern: Optional[List[str]] = None, **kwargs, ) -> str: @@ -202,8 +202,9 @@ def decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). truncate_before_pattern (`List[str]`, *optional*, defaults to `None`): A list of regular expression strings that will be used to truncate the returned string. This can be used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning diff --git a/src/transformers/models/fnet/tokenization_fnet.py b/src/transformers/models/fnet/tokenization_fnet.py index 6edcec45b5b0..7324f509a8d3 100644 --- a/src/transformers/models/fnet/tokenization_fnet.py +++ b/src/transformers/models/fnet/tokenization_fnet.py @@ -236,7 +236,7 @@ def _decode( self, token_ids: List[int], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, spaces_between_special_tokens: bool = True, **kwargs, ) -> str: @@ -269,6 +269,11 @@ def _decode( else: text = "".join(sub_texts) + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) return clean_text diff --git a/src/transformers/models/llama/tokenization_llama.py b/src/transformers/models/llama/tokenization_llama.py index 521bb854f7c1..618af846cea7 100644 --- a/src/transformers/models/llama/tokenization_llama.py +++ b/src/transformers/models/llama/tokenization_llama.py @@ -59,10 +59,17 @@ def __init__( add_bos_token=True, add_eos_token=False, decode_with_prefix_space=False, + clean_up_tokenization_spaces=False, **kwargs, ): self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs - super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs) + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) self.vocab_file = vocab_file self.add_bos_token = add_bos_token self.add_eos_token = add_eos_token diff --git a/src/transformers/models/marian/tokenization_marian.py b/src/transformers/models/marian/tokenization_marian.py index aa63b1ff88aa..96a1f47bf762 100644 --- a/src/transformers/models/marian/tokenization_marian.py +++ b/src/transformers/models/marian/tokenization_marian.py @@ -225,8 +225,9 @@ def batch_decode(self, sequences, **kwargs): List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). use_source_tokenizer (`bool`, *optional*, defaults to `False`): Whether or not to use the source tokenizer to decode sequences (only applicable in sequence-to-sequence problems). @@ -250,8 +251,9 @@ def decode(self, token_ids, **kwargs): List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). use_source_tokenizer (`bool`, *optional*, defaults to `False`): Whether or not to use the source tokenizer to decode sequences (only applicable in sequence-to-sequence problems). diff --git a/src/transformers/models/wav2vec2/tokenization_wav2vec2.py b/src/transformers/models/wav2vec2/tokenization_wav2vec2.py index 54888aea2ca2..1708dbf12512 100644 --- a/src/transformers/models/wav2vec2/tokenization_wav2vec2.py +++ b/src/transformers/models/wav2vec2/tokenization_wav2vec2.py @@ -373,7 +373,7 @@ def _decode( self, token_ids: List[int], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, group_tokens: bool = True, spaces_between_special_tokens: bool = False, output_word_offsets: Optional[bool] = False, @@ -402,6 +402,11 @@ def _decode( text = string_output["text"] + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: text = self.clean_up_tokenization(text) @@ -421,7 +426,7 @@ def batch_decode( self, sequences: Union[List[int], List[List[int]], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, output_char_offsets: bool = False, output_word_offsets: bool = False, **kwargs, @@ -434,7 +439,7 @@ def batch_decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): + clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. output_char_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output character offsets. Character offsets can be used in combination with the @@ -491,7 +496,7 @@ def decode( self, token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, output_char_offsets: bool = False, output_word_offsets: bool = False, **kwargs, @@ -507,7 +512,7 @@ def decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): + clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. output_char_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output character offsets. Character offsets can be used in combination with the @@ -887,7 +892,7 @@ def _decode( self, token_ids: List[int], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, **kwargs, ) -> str: """ @@ -905,6 +910,11 @@ def _decode( text = self.convert_tokens_to_string(result) + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) return clean_text diff --git a/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py b/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py index f3ad23a1cd74..f9a1cf631cb5 100644 --- a/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py +++ b/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py @@ -409,7 +409,7 @@ def _decode( self, token_ids: List[int], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, group_tokens: bool = True, filter_word_delimiter_token: bool = True, spaces_between_special_tokens: bool = False, @@ -438,6 +438,11 @@ def _decode( text = string_output["text"] + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: text = self.clean_up_tokenization(text) @@ -451,7 +456,7 @@ def decode( self, token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, output_char_offsets: bool = False, **kwargs, ) -> str: @@ -466,7 +471,7 @@ def decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): + clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. output_char_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output character offsets. Character offsets can be used in combination with the @@ -507,7 +512,7 @@ def batch_decode( self, sequences: Union[List[int], List[List[int]], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, output_char_offsets: bool = False, **kwargs, ) -> List[str]: @@ -519,7 +524,7 @@ def batch_decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): + clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. output_char_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output character offsets. Character offsets can be used in combination with the diff --git a/src/transformers/models/whisper/tokenization_whisper.py b/src/transformers/models/whisper/tokenization_whisper.py index 3d795e5b87e5..0160237304b6 100644 --- a/src/transformers/models/whisper/tokenization_whisper.py +++ b/src/transformers/models/whisper/tokenization_whisper.py @@ -556,7 +556,7 @@ def decode( self, token_ids, skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, output_offsets: bool = False, time_precision=0.02, decode_with_timestamps: bool = False, @@ -573,8 +573,9 @@ def decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. output_offsets (`bool`, *optional*, defaults to `False`): diff --git a/src/transformers/models/whisper/tokenization_whisper_fast.py b/src/transformers/models/whisper/tokenization_whisper_fast.py index 3110aac8b18b..fb1bf89ed606 100644 --- a/src/transformers/models/whisper/tokenization_whisper_fast.py +++ b/src/transformers/models/whisper/tokenization_whisper_fast.py @@ -266,7 +266,7 @@ def decode( self, token_ids, skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, output_offsets: bool = False, time_precision=0.02, decode_with_timestamps: bool = False, @@ -283,8 +283,9 @@ def decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. output_offsets (`bool`, *optional*, defaults to `False`): diff --git a/src/transformers/models/xlnet/tokenization_xlnet.py b/src/transformers/models/xlnet/tokenization_xlnet.py index 5ad655c41735..ec72df8c8b71 100644 --- a/src/transformers/models/xlnet/tokenization_xlnet.py +++ b/src/transformers/models/xlnet/tokenization_xlnet.py @@ -254,7 +254,7 @@ def _decode( self, token_ids: List[int], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, spaces_between_special_tokens: bool = True, **kwargs, ) -> str: @@ -284,6 +284,11 @@ def _decode( # By default, there are no spaces between special tokens text = "".join(sub_texts) + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) return clean_text diff --git a/src/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py index 3398ee30917a..a1454a744dbb 100644 --- a/src/transformers/tokenization_utils.py +++ b/src/transformers/tokenization_utils.py @@ -922,7 +922,7 @@ def _decode( self, token_ids: List[int], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, spaces_between_special_tokens: bool = True, **kwargs, ) -> str: @@ -953,6 +953,11 @@ def _decode( else: text = "".join(sub_texts) + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) return clean_text diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 66164c27788c..6162baf6f4e0 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1470,6 +1470,9 @@ def all_special_ids(self) -> List[int]: A tuple or a list of additional special tokens. Add them here to ensure they won't be split by the tokenization process. Will be associated to `self.additional_special_tokens` and `self.additional_special_tokens_ids`. + clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): + Whether or not the model should cleanup the spaces that were added when splitting the input text during the + tokenization process. """ @@ -1521,6 +1524,9 @@ def __init__(self, **kwargs): self.model_input_names = kwargs.pop("model_input_names", self.model_input_names) + # By default, cleaning tokenization spaces for both fast and slow tokenizers + self.clean_up_tokenization_spaces = kwargs.pop("clean_up_tokenization_spaces", True) + self.deprecation_warnings = ( {} ) # Use to store when we have already noticed a deprecation warning (avoid overlogging). @@ -1576,7 +1582,7 @@ def __repr__(self) -> str: f"{self.__class__.__name__}(name_or_path='{self.name_or_path}'," f" vocab_size={self.vocab_size}, model_max_length={self.model_max_length}, is_fast={self.is_fast}," f" padding_side='{self.padding_side}', truncation_side='{self.truncation_side}'," - f" special_tokens={self.special_tokens_map_extended})" + f" special_tokens={self.special_tokens_map_extended}, clean_up_tokenization_spaces={self.clean_up_tokenization_spaces})" ) def __len__(self) -> int: @@ -2112,7 +2118,7 @@ def save_pretrained( # TODO: Ensure the modified attributes (those are also in the __init__ kwargs) will give identical tokenizers # target_keys = self.init_kwargs.keys() - target_keys = ["model_max_length"] + target_keys = ["model_max_length", "clean_up_tokenization_spaces"] for k in target_keys: if hasattr(self, k): tokenizer_config[k] = getattr(self, k) @@ -3416,7 +3422,7 @@ def batch_decode( self, sequences: Union[List[int], List[List[int]], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, **kwargs, ) -> List[str]: """ @@ -3427,8 +3433,9 @@ def batch_decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces`. kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. @@ -3449,7 +3456,7 @@ def decode( self, token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, **kwargs, ) -> str: """ @@ -3463,8 +3470,9 @@ def decode( List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. - clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`): - Whether or not to clean up the tokenization spaces. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether or not to clean up the tokenization spaces. If `None`, will default to + `self.clean_up_tokenization_spaces`. kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. @@ -3485,7 +3493,7 @@ def _decode( self, token_ids: Union[int, List[int]], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, **kwargs, ) -> str: raise NotImplementedError diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py index b484464f685e..75d86d1081ac 100644 --- a/src/transformers/tokenization_utils_fast.py +++ b/src/transformers/tokenization_utils_fast.py @@ -539,7 +539,7 @@ def _decode( self, token_ids: Union[int, List[int]], skip_special_tokens: bool = False, - clean_up_tokenization_spaces: bool = True, + clean_up_tokenization_spaces: bool = None, **kwargs, ) -> str: self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False) @@ -548,6 +548,11 @@ def _decode( token_ids = [token_ids] text = self._tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens) + clean_up_tokenization_spaces = ( + clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None + else self.clean_up_tokenization_spaces + ) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) return clean_text diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index a8bdb7372b8f..53a12aedd6a9 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -3895,6 +3895,51 @@ def test_save_slow_from_fast_and_reload_fast(self): # Should not raise an error self.rust_tokenizer_class.from_pretrained(tmp_dir_2) + def test_clean_up_tokenization_spaces(self): + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + assert tokenizer.clean_up_tokenization_spaces is True + + tokens = tokenizer.encode("This shouldn't be! He'll go.") + decoded = tokenizer.decode(tokens) + assert decoded == "[CLS] this shouldn't be! he'll go. [SEP]" + + tokenizer.clean_up_tokenization_spaces = False + decoded = tokenizer.decode(tokens) + assert decoded == "[CLS] this shouldn ' t be ! he ' ll go . [SEP]" + assert decoded == tokenizer.decode(tokens, clean_up_tokenization_spaces=False) + + # Fast from slow + with tempfile.TemporaryDirectory() as tmp_dir_2: + tokenizer.save_pretrained(tmp_dir_2) + tokenizer_fast = BertTokenizerFast.from_pretrained(tmp_dir_2) + del tokenizer + + assert tokenizer_fast.clean_up_tokenization_spaces is False + decoded = tokenizer_fast.decode(tokens) + # fast and slow don't have the same output when we don't cleanup + # tokenization space. Here `be!` vs `be !` and `go.` vs `go .` + assert decoded == "[CLS] this shouldn ' t be! he ' ll go. [SEP]" + + tokenizer_fast.clean_up_tokenization_spaces = True + assert tokenizer_fast.clean_up_tokenization_spaces is True + + decoded = tokenizer_fast.decode(tokens) + assert decoded == "[CLS] this shouldn't be! he'll go. [SEP]" + + # Slow from fast + with tempfile.TemporaryDirectory() as tmp_dir_2: + tokenizer_fast.clean_up_tokenization_spaces = False + tokenizer_fast.save_pretrained(tmp_dir_2) + tokenizer = BertTokenizer.from_pretrained(tmp_dir_2) + + assert tokenizer_fast.clean_up_tokenization_spaces is False + decoded = tokenizer.decode(tokens) + assert decoded == "[CLS] this shouldn ' t be ! he ' ll go . [SEP]" + + tokenizer.clean_up_tokenization_spaces = True + decoded = tokenizer.decode(tokens) + assert decoded == "[CLS] this shouldn't be! he'll go. [SEP]" + class TokenizerUtilTester(unittest.TestCase): def test_cached_files_are_used_when_internet_is_down(self): From fc3f40d999b32d9cb547d0db2911d33e6dc16074 Mon Sep 17 00:00:00 2001 From: Nolwenn Bernard <28621493+NoB0@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:09:57 +0200 Subject: [PATCH 215/277] Hyperparameter search reporting to W&B (#22440) Fixes #22429 --- src/transformers/integrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/integrations.py b/src/transformers/integrations.py index 52e4d92148ec..a93eab958197 100644 --- a/src/transformers/integrations.py +++ b/src/transformers/integrations.py @@ -478,7 +478,7 @@ def run_hp_search_wandb(trainer, n_trials: int, direction: str, **kwargs) -> Bes break if not reporting_to_wandb: trainer.add_callback(WandbCallback()) - trainer.args.report_to = "wandb" + trainer.args.report_to = ["wandb"] best_trial = {"run_id": None, "objective": None, "hyperparameters": None} sweep_id = kwargs.pop("sweep_id", None) project = kwargs.pop("project", None) From 1ead5017e685392ccc208c3633b763da550ffbaa Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:13:00 +0200 Subject: [PATCH 216/277] [`bnb`] fix bnb failing test (#22439) * fix bnb failing test * fix * fix * fixup --- tests/mixed_int8/test_mixed_int8.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/mixed_int8/test_mixed_int8.py b/tests/mixed_int8/test_mixed_int8.py index 968e2d346c13..a73f689a8a94 100644 --- a/tests/mixed_int8/test_mixed_int8.py +++ b/tests/mixed_int8/test_mixed_int8.py @@ -384,9 +384,8 @@ def test_multi_gpu_loading(self): Let's just try to load a model on 2 GPUs and see if it works. The model we test has ~2GB of total, 3GB should suffice """ - memory_mapping = {0: "1GB", 1: "2GB"} model_parallel = AutoModelForCausalLM.from_pretrained( - self.model_name, load_in_8bit=True, max_memory=memory_mapping, device_map="auto" + self.model_name, load_in_8bit=True, device_map="balanced" ) # Check correct device map From 5ff48083514b9e34dd8c950a5ddde483a8af33ea Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:35:30 +0200 Subject: [PATCH 217/277] [`Generate`] Add conditional generation for multimodal models (#22424) * add conditional generation * add comments --- src/transformers/generation/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 1ccb41292158..ee92e51a19a5 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -1288,6 +1288,10 @@ def generate( model_kwargs=model_kwargs, device=inputs_tensor.device, ) + + # conditional generation for multi-modal models. + if "input_ids" in model_kwargs and model_input_name == "pixel_values": + input_ids = torch.cat([input_ids, model_kwargs.pop("input_ids")], dim=-1) else: input_ids = inputs_tensor if model_input_name == "input_ids" else model_kwargs.pop("input_ids") From 93187cef8332cc66d105518b96541714e18ebf81 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 29 Mar 2023 09:46:30 -0400 Subject: [PATCH 218/277] Don't hard error when cache version can't be converted to int (#22427) --- src/transformers/utils/hub.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/utils/hub.py b/src/transformers/utils/hub.py index db00878c9ae4..6bedd51509de 100644 --- a/src/transformers/utils/hub.py +++ b/src/transformers/utils/hub.py @@ -1082,7 +1082,10 @@ def move_cache(cache_dir=None, new_cache_dir=None, token=None): cache_version = 0 else: with open(cache_version_file) as f: - cache_version = int(f.read()) + try: + cache_version = int(f.read()) + except ValueError: + cache_version = 0 cache_is_not_empty = os.path.isdir(TRANSFORMERS_CACHE) and len(os.listdir(TRANSFORMERS_CACHE)) > 0 From 80b7e6cef91f647963d646963eff1c9eef280f6d Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 29 Mar 2023 16:16:23 +0200 Subject: [PATCH 219/277] Use real tokenizers if tiny version(s) creation has issue(s) (#22428) Fix some tiny model creation issues Co-authored-by: ydshieh --- .github/workflows/update_tiny_models.yml | 2 +- .../test_modeling_bigbird_pegasus.py | 9 + .../test_modeling_xlm_roberta_xl.py | 9 + tests/models/xmod/test_modeling_xmod.py | 9 + tests/utils/tiny_model_summary.json | 256 +++++++++++------- utils/create_dummy_models.py | 184 ++++++++++--- 6 files changed, 333 insertions(+), 136 deletions(-) diff --git a/.github/workflows/update_tiny_models.yml b/.github/workflows/update_tiny_models.yml index c02196077508..1ed8271f8ab2 100644 --- a/.github/workflows/update_tiny_models.yml +++ b/.github/workflows/update_tiny_models.yml @@ -1,4 +1,4 @@ -name: Self-hosted runner (push) +name: Update Tiny Models on: push: diff --git a/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py b/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py index d7a8e6302d84..88bc457c3619 100644 --- a/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py +++ b/tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py @@ -268,6 +268,15 @@ class BigBirdPegasusModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineT # Also torchscript is not an important feature to have in the beginning. test_torchscript = False + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "QAPipelineTests" and not tokenizer_name.endswith("Fast"): + return True + + return False + # overwrite from GenerationTesterMixin to solve problem # with conflicting random seeds def _get_input_ids_and_config(self): diff --git a/tests/models/xlm_roberta_xl/test_modeling_xlm_roberta_xl.py b/tests/models/xlm_roberta_xl/test_modeling_xlm_roberta_xl.py index 2b7e77ea021c..7ec84c9b1f2b 100644 --- a/tests/models/xlm_roberta_xl/test_modeling_xlm_roberta_xl.py +++ b/tests/models/xlm_roberta_xl/test_modeling_xlm_roberta_xl.py @@ -387,6 +387,15 @@ class XLMRobertaXLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTes else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "QAPipelineTests" and not tokenizer_name.endswith("Fast"): + return True + + return False + def setUp(self): self.model_tester = XLMRobertaXLModelTester(self) self.config_tester = ConfigTester(self, config_class=XLMRobertaXLConfig, hidden_size=37) diff --git a/tests/models/xmod/test_modeling_xmod.py b/tests/models/xmod/test_modeling_xmod.py index 35d226036f39..5845dee7448d 100644 --- a/tests/models/xmod/test_modeling_xmod.py +++ b/tests/models/xmod/test_modeling_xmod.py @@ -384,6 +384,15 @@ class XmodModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin else {} ) + # TODO: Fix the failed tests + def is_pipeline_test_to_skip( + self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name + ): + if pipeline_test_casse_name == "QAPipelineTests" and not tokenizer_name.endswith("Fast"): + return True + + return False + def setUp(self): self.model_tester = XmodModelTester(self) self.config_tester = ConfigTester(self, config_class=XmodConfig, hidden_size=37) diff --git a/tests/utils/tiny_model_summary.json b/tests/utils/tiny_model_summary.json index 952b599db5f7..f791251f7c2a 100644 --- a/tests/utils/tiny_model_summary.json +++ b/tests/utils/tiny_model_summary.json @@ -21,6 +21,7 @@ }, "AlbertForMaskedLM": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], @@ -28,10 +29,11 @@ "AlbertForMaskedLM", "TFAlbertForMaskedLM" ], - "sha": "75ab12f94d4a1edd9610636547c5fb515e240e2b" + "sha": "d29de71ac29e1019c3a7762f7357f750730cb037" }, "AlbertForMultipleChoice": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], @@ -39,10 +41,11 @@ "AlbertForMultipleChoice", "TFAlbertForMultipleChoice" ], - "sha": "ba1531e4373cccce03195928b3ba2f6825311980" + "sha": "242aecce6a589a2964c0f695621fa22a83751579" }, "AlbertForPreTraining": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], @@ -50,10 +53,11 @@ "AlbertForPreTraining", "TFAlbertForPreTraining" ], - "sha": "6022449842a83d9cea298c4fbaf1e1e1c0db3568" + "sha": "41330be4b271687f4d88ddc96346c12aa11de983" }, "AlbertForQuestionAnswering": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], @@ -61,10 +65,11 @@ "AlbertForQuestionAnswering", "TFAlbertForQuestionAnswering" ], - "sha": "1b6584d6a267dae8ff20b9f89e2b424a7972fb45" + "sha": "040b81c15f437f4722349dc5b41fccd17ebd7fdc" }, "AlbertForSequenceClassification": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], @@ -72,10 +77,11 @@ "AlbertForSequenceClassification", "TFAlbertForSequenceClassification" ], - "sha": "1e709531344ee0e4a34777c79507a07b69130958" + "sha": "39c1a0e2c1c2623106d3211d751e9b32f23a91a0" }, "AlbertForTokenClassification": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], @@ -83,10 +89,11 @@ "AlbertForTokenClassification", "TFAlbertForTokenClassification" ], - "sha": "f6c0d721d6d9f0751ab975148932948d5853fcc8" + "sha": "359c3f4a311a4053a6f6d6a880db5f82c8e3ff1f" }, "AlbertModel": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], @@ -94,7 +101,7 @@ "AlbertModel", "TFAlbertModel" ], - "sha": "62974edf8b7246a943f6ecc8a3f7bfca052351ff" + "sha": "34a63314686b64aaeb595ddb95006f1ff2ffda17" }, "AlignModel": { "tokenizer_classes": [ @@ -318,133 +325,146 @@ }, "BigBirdForCausalLM": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdForCausalLM" ], - "sha": "04578a05f11c0006e4e5deaf38b48889a8dc8f4f" + "sha": "5c7a487af5248d9c01b45d5481b7d7bb9b36e1b5" }, "BigBirdForMaskedLM": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdForMaskedLM" ], - "sha": "52b8e93488b5d8235711543d4671ea08ea3f9560" + "sha": "476ef8225c0f69270b577706ad4f1dda13e4dde5" }, "BigBirdForMultipleChoice": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdForMultipleChoice" ], - "sha": "56459f9bcde6a36870e4d743295f6ce69ba5fc7b" + "sha": "cf93eaa1019987112c171a407745bc183a20513a" }, "BigBirdForPreTraining": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdForPreTraining" ], - "sha": "49f55d7252dd9151722b330fa02073c6d809c55e" + "sha": "5fb9efa13334431e7c186a9fa314b89c4a1eee72" }, "BigBirdForQuestionAnswering": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdForQuestionAnswering" ], - "sha": "a5b3c8567610d4dde63c282d6fb6fd2ec04cbb39" + "sha": "f82f88bd71fba819a8ffb0692915d3529e705417" }, "BigBirdForSequenceClassification": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdForSequenceClassification" ], - "sha": "2b29389f623fa7af3ffb08c51e9bcbda270ae9ee" + "sha": "ea398090858f9af93b54fc9a8d65cfed78ac27ff" }, "BigBirdForTokenClassification": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdForTokenClassification" ], - "sha": "beda63d67d07b133e603f51aea6b84cae29b9ea7" + "sha": "2cdea118999fa58ba9fb0162d99e2ffa146c3df1" }, "BigBirdModel": { "tokenizer_classes": [ + "BigBirdTokenizer", "BigBirdTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdModel" ], - "sha": "814adde7ccd69821684a0c6124401f0c180d700c" + "sha": "9c55989f31df156194e6997606fb14d9897e0300" }, "BigBirdPegasusForCausalLM": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdPegasusForCausalLM" ], - "sha": "e1a5b87220073127f718fec558cbc86795b6ed61" + "sha": "49bc8816c666dee32e27cd8e00136b604eb85243" }, "BigBirdPegasusForConditionalGeneration": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdPegasusForConditionalGeneration" ], - "sha": "40fad528589229426174241a641034f1f971a2b7" + "sha": "e791aa6d1af5a76ca0926d95b1f28bd2d8adf376" }, "BigBirdPegasusForQuestionAnswering": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdPegasusForQuestionAnswering" ], - "sha": "18d836e06a02bd1dc36af9a5eeaf3d326b1d368a" + "sha": "7650e076713ca707a37062adc8c9c1cd60dad7c7" }, "BigBirdPegasusForSequenceClassification": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdPegasusForSequenceClassification" ], - "sha": "a451bdb3f36fb76af06a51274d19fd88729443e6" + "sha": "02500e8ebd9c53528750013fb963fbdc2be34034" }, "BigBirdPegasusModel": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "BigBirdPegasusModel" ], - "sha": "3ece62a543ced3755e5ae239bcc3a4a5a6b75dd4" + "sha": "b07c5304dfba673cf8b9cf5cd1aa45fbfea1c2f3" }, "BioGptForCausalLM": { "tokenizer_classes": [ @@ -1240,6 +1260,7 @@ }, "DebertaV2ForMaskedLM": { "tokenizer_classes": [ + "DebertaV2Tokenizer", "DebertaV2TokenizerFast" ], "processor_classes": [], @@ -1247,20 +1268,22 @@ "DebertaV2ForMaskedLM", "TFDebertaV2ForMaskedLM" ], - "sha": "9089b6afa8f66fd16503fca2b7c54b50c2195123" + "sha": "a053dedc2cdf32918a84277cb0c05186604496a5" }, "DebertaV2ForMultipleChoice": { "tokenizer_classes": [ + "DebertaV2Tokenizer", "DebertaV2TokenizerFast" ], "processor_classes": [], "model_classes": [ "DebertaV2ForMultipleChoice" ], - "sha": "7acf4b7415b2869e225a5ed82e68f1c0374e9668" + "sha": "07e39f520ce239b39ef8cb24cd7874d06c791063" }, "DebertaV2ForQuestionAnswering": { "tokenizer_classes": [ + "DebertaV2Tokenizer", "DebertaV2TokenizerFast" ], "processor_classes": [], @@ -1268,10 +1291,11 @@ "DebertaV2ForQuestionAnswering", "TFDebertaV2ForQuestionAnswering" ], - "sha": "17ef18fefddc0ec61a972eea06af430059ae3759" + "sha": "9cecb3a7fc6b95099122283644ea1f8ced287d1b" }, "DebertaV2ForSequenceClassification": { "tokenizer_classes": [ + "DebertaV2Tokenizer", "DebertaV2TokenizerFast" ], "processor_classes": [], @@ -1279,10 +1303,11 @@ "DebertaV2ForSequenceClassification", "TFDebertaV2ForSequenceClassification" ], - "sha": "1ef484d43eb15ac6b1f8be393c3d59bea2267dc9" + "sha": "df9ea1f5c0f2ccd139b21cfb3963a5a5ebfb5b81" }, "DebertaV2ForTokenClassification": { "tokenizer_classes": [ + "DebertaV2Tokenizer", "DebertaV2TokenizerFast" ], "processor_classes": [], @@ -1290,10 +1315,11 @@ "DebertaV2ForTokenClassification", "TFDebertaV2ForTokenClassification" ], - "sha": "5c4e629b5b03957a546f7f76c31f6887f99fc17c" + "sha": "51fe01989df38a540ac1abca5ee71a51365defd5" }, "DebertaV2Model": { "tokenizer_classes": [ + "DebertaV2Tokenizer", "DebertaV2TokenizerFast" ], "processor_classes": [], @@ -1301,7 +1327,7 @@ "DebertaV2Model", "TFDebertaV2Model" ], - "sha": "a1945cc2bb1ef207f8fdeb4ea146711ade1db77a" + "sha": "211df4bd1a4a9b66c97af3f9231a5d2af8de7b9f" }, "DeformableDetrForObjectDetection": { "tokenizer_classes": [], @@ -1881,83 +1907,91 @@ }, "FNetForMaskedLM": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetForMaskedLM" ], - "sha": "235602782fb4b4ff0291c999cc174b4a257f1e7f" + "sha": "91eaae1eac894af5d96c0221ec9bcef7f1af41c8" }, "FNetForMultipleChoice": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetForMultipleChoice" ], - "sha": "d84b9ee07323895465a29f234e9109b66fd623cf" + "sha": "c15d98d5f7a6f3ef3099b1257949bee208d5466e" }, "FNetForNextSentencePrediction": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetForNextSentencePrediction" ], - "sha": "9b11a763f599a95c3dff8e4255cd952a04101a65" + "sha": "c59440b44d07d61fc45a90ded7fc11d6f25b143d" }, "FNetForPreTraining": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetForPreTraining" ], - "sha": "ed6ec245f3a8f7b53c7b09b020cfae1f8c4aaf7d" + "sha": "c05f55ccfb2f2533babd3c6e99de7749bc8081da" }, "FNetForQuestionAnswering": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetForQuestionAnswering" ], - "sha": "c77c577acae60cd268b0eebdbffcbd75f8e31141" + "sha": "47788e49dd435653fa2aa4b3ccae3572a870758e" }, "FNetForSequenceClassification": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetForSequenceClassification" ], - "sha": "6e6dbab691e5ec18e04b98514f1656dc3a842192" + "sha": "a3049b896ea6c5a32c364989c3afe604ee58b9fc" }, "FNetForTokenClassification": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetForTokenClassification" ], - "sha": "b27e341994ef7913dcdd72326d3475c9668d07d5" + "sha": "3bcdafca57d544bb81e2f7eead1e512c168582fc" }, "FNetModel": { "tokenizer_classes": [ + "FNetTokenizer", "FNetTokenizerFast" ], "processor_classes": [], "model_classes": [ "FNetModel" ], - "sha": "38041395aa488da543dec9ed86318d9ec4d4839e" + "sha": "48fa66de37df126504db3b658806135eb877f505" }, "FSMTForConditionalGeneration": { "tokenizer_classes": [ @@ -2828,23 +2862,25 @@ }, "LongT5ForConditionalGeneration": { "tokenizer_classes": [ + "T5Tokenizer", "T5TokenizerFast" ], "processor_classes": [], "model_classes": [ "LongT5ForConditionalGeneration" ], - "sha": "3c83684439875f33fc0f79d164e7b446245cbc74" + "sha": "c685cbbe706ad5c9a28689631765726a1874dcc7" }, "LongT5Model": { "tokenizer_classes": [ + "T5Tokenizer", "T5TokenizerFast" ], "processor_classes": [], "model_classes": [ "LongT5Model" ], - "sha": "3b825c077d41103476d88bd93a6a9af52194b99e" + "sha": "6b468e55e2490565e6155690201086ac00c72062" }, "LongformerForMaskedLM": { "tokenizer_classes": [ @@ -3042,7 +3078,7 @@ "model_classes": [ "MBartForCausalLM" ], - "sha": "e0f8bdbcc8afd38ef9a9d78ad96d3e93ddd19d71" + "sha": "a45044f8056328d20a764356eca3d0746a7a195e" }, "MBartForConditionalGeneration": { "tokenizer_classes": [ @@ -3054,7 +3090,7 @@ "MBartForConditionalGeneration", "TFMBartForConditionalGeneration" ], - "sha": "9d2a48747c3671ef7a9b0c202d79c677b6d6f272" + "sha": "171e918962d6c0ee56c6b070858e19e16c8dd09f" }, "MBartForQuestionAnswering": { "tokenizer_classes": [ @@ -3065,7 +3101,7 @@ "model_classes": [ "MBartForQuestionAnswering" ], - "sha": "0ad08bf0c1c39d358b92b38f75cbe04609167c3f" + "sha": "1ee08565d24777335595e0d2940e454abdcff731" }, "MBartForSequenceClassification": { "tokenizer_classes": [ @@ -3076,7 +3112,7 @@ "model_classes": [ "MBartForSequenceClassification" ], - "sha": "05ed9ea84c02f4a5347f5c235d38f9df891cb811" + "sha": "53e9c88ecfa2475d27afe099ffa7a8bcdb7ef7e4" }, "MBartModel": { "tokenizer_classes": [ @@ -3088,7 +3124,7 @@ "MBartModel", "TFMBartModel" ], - "sha": "21e9aa6eff4ae280e299ac33c94f1a317300571b" + "sha": "2d492b34d69dd63b411990d5c8bb692fd637e91c" }, "MCTCTForCTC": { "tokenizer_classes": [], @@ -3767,63 +3803,69 @@ }, "NystromformerForMaskedLM": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], "model_classes": [ "NystromformerForMaskedLM" ], - "sha": "a21609ab92ea2ea438a4c78f9c2faf0594681fb2" + "sha": "37036847783f1e65e81ecd43803270a1ecb276f3" }, "NystromformerForMultipleChoice": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], "model_classes": [ "NystromformerForMultipleChoice" ], - "sha": "417bcc3c7636f1b0a5cbd7371ddce566c302e52d" + "sha": "42a077d5ab6830e20560466eaccc525eff10c3ae" }, "NystromformerForQuestionAnswering": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], "model_classes": [ "NystromformerForQuestionAnswering" ], - "sha": "206c4a09179f8eda76dab66a0b8d5bd61e031a79" + "sha": "1cfaf79051731824db4f09989f093f87f4fceec5" }, "NystromformerForSequenceClassification": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], "model_classes": [ "NystromformerForSequenceClassification" ], - "sha": "0e0f7cc4d7cf9a759ac5d1a2ebbcc6026b4c7df3" + "sha": "d75231203066df41e9b6b25dbee9ad40e8515c18" }, "NystromformerForTokenClassification": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], "model_classes": [ "NystromformerForTokenClassification" ], - "sha": "3b140a528dd8d259f4edf229430edc8f7633473e" + "sha": "5a499dc96e106bf41fc9166f2ad06527ec7ca14e" }, "NystromformerModel": { "tokenizer_classes": [ + "AlbertTokenizer", "AlbertTokenizerFast" ], "processor_classes": [], "model_classes": [ "NystromformerModel" ], - "sha": "761c4db14685ba334ec012a5075847b3439da7d1" + "sha": "2b6adb37ec473b15d71e2eb459acea08df6940ce" }, "OPTForCausalLM": { "tokenizer_classes": [ @@ -4001,16 +4043,18 @@ }, "PegasusForCausalLM": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "PegasusForCausalLM" ], - "sha": "bedf3a4a4188e44309417b70243f210c68b0a6f8" + "sha": "6e685a698302a3ba33e5379d3a37eb0bc1ae2f70" }, "PegasusForConditionalGeneration": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], @@ -4018,10 +4062,11 @@ "PegasusForConditionalGeneration", "TFPegasusForConditionalGeneration" ], - "sha": "cacda18e0c679447ca1f38895fc08650df13d571" + "sha": "15e58ee2ebc14b6e80ef2891259057ee5f049be2" }, "PegasusModel": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], @@ -4029,27 +4074,29 @@ "PegasusModel", "TFPegasusModel" ], - "sha": "a3e9e2ccb99adb9ac068f7ab74946bbdbfed52ec" + "sha": "fa36b24523db411ef77903453346b8be81ef73fe" }, "PegasusXForConditionalGeneration": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "PegasusXForConditionalGeneration" ], - "sha": "7edff1791c2c20a6983202836a5b9c732f64631e" + "sha": "7588a8120f26a36c1687c14bdf1e9f9656891c1a" }, "PegasusXModel": { "tokenizer_classes": [ + "PegasusTokenizer", "PegasusTokenizerFast" ], "processor_classes": [], "model_classes": [ "PegasusXModel" ], - "sha": "921751ff0cf59e7cfe909fa6803b5d5ceb0d3059" + "sha": "a0bdff627416ac3c39c22d081f5d88d8b8fd99cc" }, "PerceiverForImageClassificationConvProcessing": { "tokenizer_classes": [ @@ -4250,6 +4297,7 @@ }, "RemBertForCausalLM": { "tokenizer_classes": [ + "RemBertTokenizer", "RemBertTokenizerFast" ], "processor_classes": [], @@ -4257,10 +4305,11 @@ "RemBertForCausalLM", "TFRemBertForCausalLM" ], - "sha": "0b85a3f867b04078e2b3070dbb7033475946415f" + "sha": "8d9ae3d74a0e0a8958b4ee8c9dca3632abf52ef9" }, "RemBertForMaskedLM": { "tokenizer_classes": [ + "RemBertTokenizer", "RemBertTokenizerFast" ], "processor_classes": [], @@ -4268,10 +4317,11 @@ "RemBertForMaskedLM", "TFRemBertForMaskedLM" ], - "sha": "706f273263519d12cb35736b3b03a96908f752bf" + "sha": "b7c27d01e1cc3bef9ddd6a78627d700b3bffd759" }, "RemBertForMultipleChoice": { "tokenizer_classes": [ + "RemBertTokenizer", "RemBertTokenizerFast" ], "processor_classes": [], @@ -4279,10 +4329,11 @@ "RemBertForMultipleChoice", "TFRemBertForMultipleChoice" ], - "sha": "e03cd3b6eddb3ad9045e4d6d1053812a393a9417" + "sha": "2fe192677b9740cf24dd559339d46925e8ac23d4" }, "RemBertForQuestionAnswering": { "tokenizer_classes": [ + "RemBertTokenizer", "RemBertTokenizerFast" ], "processor_classes": [], @@ -4290,10 +4341,11 @@ "RemBertForQuestionAnswering", "TFRemBertForQuestionAnswering" ], - "sha": "882925e63c946dfb9679cd80a4c4909fbf1e4f2c" + "sha": "22b8ba44681b96292a1cf7f6df4ba6bb7937ec6e" }, "RemBertForSequenceClassification": { "tokenizer_classes": [ + "RemBertTokenizer", "RemBertTokenizerFast" ], "processor_classes": [], @@ -4301,10 +4353,11 @@ "RemBertForSequenceClassification", "TFRemBertForSequenceClassification" ], - "sha": "d315ecf3be15d52df6d7a137702f896265f112b1" + "sha": "20f3e89341ea15266d2685a8798142fba03c3f98" }, "RemBertForTokenClassification": { "tokenizer_classes": [ + "RemBertTokenizer", "RemBertTokenizerFast" ], "processor_classes": [], @@ -4312,10 +4365,11 @@ "RemBertForTokenClassification", "TFRemBertForTokenClassification" ], - "sha": "622e82ab7284c36acb1260a171623b650f0a8f52" + "sha": "15712ff753708da3cf0550e76e73a5d0bba7784e" }, "RemBertModel": { "tokenizer_classes": [ + "RemBertTokenizer", "RemBertTokenizerFast" ], "processor_classes": [], @@ -4323,7 +4377,7 @@ "RemBertModel", "TFRemBertModel" ], - "sha": "7dc1b9ee6493a52c377a4ceda0e6a24780c33281" + "sha": "59cc6d099b1ded0aaead8684457415b129f79e86" }, "ResNetBackbone": { "tokenizer_classes": [], @@ -5038,26 +5092,29 @@ }, "SwitchTransformersForConditionalGeneration": { "tokenizer_classes": [ + "T5Tokenizer", "T5TokenizerFast" ], "processor_classes": [], "model_classes": [ "SwitchTransformersForConditionalGeneration" ], - "sha": "332cf96ed681e6cadb5b0788d1925224b62f9cf4" + "sha": "c8fcd2bb735894c78db7f1e5b51afc78aced7adb" }, "SwitchTransformersModel": { "tokenizer_classes": [ + "T5Tokenizer", "T5TokenizerFast" ], "processor_classes": [], "model_classes": [ "SwitchTransformersModel" ], - "sha": "a367b372751ab0e647724c25fc3698f5d46bce20" + "sha": "275bbf6d389bfd0540b9f824c609c6b22a577328" }, "T5ForConditionalGeneration": { "tokenizer_classes": [ + "T5Tokenizer", "T5TokenizerFast" ], "processor_classes": [], @@ -5065,10 +5122,11 @@ "T5ForConditionalGeneration", "TFT5ForConditionalGeneration" ], - "sha": "718845103ac90ab0c0594b6a7149ca10e5680f7e" + "sha": "593fd6072a4e265f5cc73b1973cd8af76b261f29" }, "T5Model": { "tokenizer_classes": [ + "T5Tokenizer", "T5TokenizerFast" ], "processor_classes": [], @@ -5076,7 +5134,7 @@ "T5Model", "TFT5Model" ], - "sha": "e47412452747fd8a9921371fa8f7cc92c9be973f" + "sha": "eb3d20dda0ba77c1de618d78116a1a0c784c515c" }, "TableTransformerForObjectDetection": { "tokenizer_classes": [], @@ -5776,6 +5834,7 @@ }, "XGLMForCausalLM": { "tokenizer_classes": [ + "XGLMTokenizer", "XGLMTokenizerFast" ], "processor_classes": [], @@ -5783,10 +5842,11 @@ "TFXGLMForCausalLM", "XGLMForCausalLM" ], - "sha": "95dc5c571119b6bf7d7c58e98897be4c724ca4ee" + "sha": "d5381ce297c249d559937c6bb6316cf1fdad2613" }, "XGLMModel": { "tokenizer_classes": [ + "XGLMTokenizer", "XGLMTokenizerFast" ], "processor_classes": [], @@ -5794,7 +5854,7 @@ "TFXGLMModel", "XGLMModel" ], - "sha": "fcc8ad648ae1561cb4b35a1fd31c68b58296ef00" + "sha": "2b5cef167822cfaa558d259af1722e2f785cd3d5" }, "XLMForMultipleChoice": { "tokenizer_classes": [ @@ -5853,73 +5913,80 @@ }, "XLMRobertaXLForCausalLM": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XLMRobertaXLForCausalLM" ], - "sha": "54d7d901a31fcea581772a62a574f88ea994d931" + "sha": "fc05408e5b33a31638476ef337719dfbb7615ef3" }, "XLMRobertaXLForMaskedLM": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XLMRobertaXLForMaskedLM" ], - "sha": "b9b8090515146aa703fc30da03f3c0995ad2ac58" + "sha": "e96f198eede757e5ae2c87632fdcfb341073ef6e" }, "XLMRobertaXLForMultipleChoice": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XLMRobertaXLForMultipleChoice" ], - "sha": "c6e7ef0a2d640d14f5b0333ffcd32fb2fb83312d" + "sha": "52732625f1bfbbb7cb4ba1cf0963de596d81822d" }, "XLMRobertaXLForQuestionAnswering": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XLMRobertaXLForQuestionAnswering" ], - "sha": "4e433426b85bf8c1cea9261e91bd52d78d7bf397" + "sha": "da388fdd2d28e0757eb0c2b2c612a8ff03af2223" }, "XLMRobertaXLForSequenceClassification": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XLMRobertaXLForSequenceClassification" ], - "sha": "0b8361e84d0d75c0c7540e8dfcc339d37f683800" + "sha": "980721187633bcf21ac0b8edbed933527f4611df" }, "XLMRobertaXLForTokenClassification": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XLMRobertaXLForTokenClassification" ], - "sha": "70c4a472222415f8611c25ac7eea48944a6fa0aa" + "sha": "37a97280faf6fef0bd946d3934d77a1b60fbf473" }, "XLMRobertaXLModel": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XLMRobertaXLModel" ], - "sha": "df4e5bb09f2fa1e5582dc9424882e544c8b28aed" + "sha": "8fbeb39a984912e47f5d24a31be61639031a0fc3" }, "XLMWithLMHeadModel": { "tokenizer_classes": [ @@ -5942,7 +6009,7 @@ "TFXLNetForMultipleChoice", "XLNetForMultipleChoice" ], - "sha": "874306eaf380012b985180563bf140d604272f40" + "sha": "8bb7e28d0cd1e93154d3232baf5e9c79acaf9f1a" }, "XLNetForQuestionAnsweringSimple": { "tokenizer_classes": [ @@ -5954,7 +6021,7 @@ "TFXLNetForQuestionAnsweringSimple", "XLNetForQuestionAnsweringSimple" ], - "sha": "8c6ac70e1946da872e5f93853247728d672a1dee" + "sha": "fabd06a45d947f3d46f1b8dce2186cf3b27776dc" }, "XLNetForSequenceClassification": { "tokenizer_classes": [ @@ -5966,7 +6033,7 @@ "TFXLNetForSequenceClassification", "XLNetForSequenceClassification" ], - "sha": "3bba8894ba1d8e34cd807fd5a0f858449abf65b4" + "sha": "e3c194f24537ebf2c474ade60becb9397696edec" }, "XLNetForTokenClassification": { "tokenizer_classes": [ @@ -5978,7 +6045,7 @@ "TFXLNetForTokenClassification", "XLNetForTokenClassification" ], - "sha": "70598399e95e6df689fe5459ef9f016e55cc05ee" + "sha": "16aa15029aa667046d504c4a88ceddfdd5b5fb40" }, "XLNetLMHeadModel": { "tokenizer_classes": [ @@ -5990,7 +6057,7 @@ "TFXLNetLMHeadModel", "XLNetLMHeadModel" ], - "sha": "fef32495d187c73201ba4e2854559bcc68e41e22" + "sha": "c9a98cc982a16ca162832a8cbea25116479bb938" }, "XLNetModel": { "tokenizer_classes": [ @@ -6002,77 +6069,84 @@ "TFXLNetModel", "XLNetModel" ], - "sha": "bebc65e9a3da5c0007713a61f6719293d361baa3" + "sha": "1d6e231942135faf32b8d9a97773d8f6c85ca561" }, "XmodForCausalLM": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XmodForCausalLM" ], - "sha": "daf792dff8fef2f3d99eb5ee63b206032b3f69d7" + "sha": "c6b746071f2f067099a8fb4f57ce3c27a7e4b67d" }, "XmodForMaskedLM": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XmodForMaskedLM" ], - "sha": "b7c020898ef638ed4c2f420431b4efc0075c0e32" + "sha": "e1085818f4ed3c6073b2038635e5f3061208923d" }, "XmodForMultipleChoice": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XmodForMultipleChoice" ], - "sha": "1fce9673e9f557d3204d504ebb86e285b20937f8" + "sha": "c63042cdf196be3fed846421b345d439b2483f69" }, "XmodForQuestionAnswering": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XmodForQuestionAnswering" ], - "sha": "fc9ebdfeb481281375c25d585569404cc48b02da" + "sha": "75acd3071fae9978c82618cd0f090c87aabc1f23" }, "XmodForSequenceClassification": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XmodForSequenceClassification" ], - "sha": "bba89c4e18b6a29a1865462a75cb8bde12e7cc0c" + "sha": "523a16570be048618913ac17ccd00d343bcb5e99" }, "XmodForTokenClassification": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XmodForTokenClassification" ], - "sha": "efe3dad234d5a60f1165887d8b4bb1627e271508" + "sha": "a0f0a02732b4579670dad11a69ae244ebd777b49" }, "XmodModel": { "tokenizer_classes": [ + "XLMRobertaTokenizer", "XLMRobertaTokenizerFast" ], "processor_classes": [], "model_classes": [ "XmodModel" ], - "sha": "62f3ca59e02a9637b12ce81566f1996c16fad872" + "sha": "bc286de0035450e7dcd6bcce78098a967b9c2b6c" }, "YolosForObjectDetection": { "tokenizer_classes": [], @@ -6102,7 +6176,7 @@ "model_classes": [ "YosoForMaskedLM" ], - "sha": "4ff7ab217e6d05d40ae8afd37df1a0bf5c6ab25f" + "sha": "cb291bedcbec199ea195f086e3ebea6fab026bba" }, "YosoForMultipleChoice": { "tokenizer_classes": [ @@ -6112,7 +6186,7 @@ "model_classes": [ "YosoForMultipleChoice" ], - "sha": "c42bd94e4563cdfeb28e07aef80e801188706d9d" + "sha": "cf2d3a3f0628bc9d0da68ea8de26b12016453fee" }, "YosoForQuestionAnswering": { "tokenizer_classes": [ @@ -6122,7 +6196,7 @@ "model_classes": [ "YosoForQuestionAnswering" ], - "sha": "284f170b902c65ee257a2e8f255fa672cb4700c5" + "sha": "e8c3091f674588adfa3371b3de0427a9b39dd03f" }, "YosoForSequenceClassification": { "tokenizer_classes": [ @@ -6132,7 +6206,7 @@ "model_classes": [ "YosoForSequenceClassification" ], - "sha": "2e0a37a4dd12cee00eb15e4ecd0de4231a638966" + "sha": "88132cbaa1a9a87f65b6f9813c388011377f18cf" }, "YosoForTokenClassification": { "tokenizer_classes": [ @@ -6142,7 +6216,7 @@ "model_classes": [ "YosoForTokenClassification" ], - "sha": "50dd3fe46f28f0efaacf8aa311b09f67b10edb74" + "sha": "fd2219856608d3dba70dc7b1a06af629903dec31" }, "YosoModel": { "tokenizer_classes": [ @@ -6152,6 +6226,6 @@ "model_classes": [ "YosoModel" ], - "sha": "6b42cca4d4f9204a3aa1197664663656719095cb" + "sha": "e144d9f1fe39c21eda1177702640e126892605ce" } } \ No newline at end of file diff --git a/utils/create_dummy_models.py b/utils/create_dummy_models.py index 9fb53df0ca26..cba13c12d3c6 100644 --- a/utils/create_dummy_models.py +++ b/utils/create_dummy_models.py @@ -405,7 +405,11 @@ def get_tiny_config(config_class, model_class=None, **model_tester_kwargs): for _tester_classes in models_to_model_testers.values(): tester_classes.extend(_tester_classes) if len(tester_classes) > 0: - model_tester_class = sorted(tester_classes, key=lambda x: x.__name__)[0] + # sort with the length of the class names first, then the alphabetical order + # This is to avoid `T5EncoderOnlyModelTest` is used instead of `T5ModelTest`, which has + # `is_encoder_decoder=False` and causes some pipeline tests failing (also failures in `Optimum` CI). + # TODO: More fine grained control of the desired tester class. + model_tester_class = sorted(tester_classes, key=lambda x: (len(x.__name__), x.__name__))[0] except ModuleNotFoundError: error = f"Tiny config not created for {model_type} - cannot find the testing module from the model name." raise ValueError(error) @@ -484,22 +488,68 @@ def convert_processors(processors, tiny_config, output_folder, result): This method should not fail: we catch the errors and put them in `result["warnings"]` with descriptive messages. """ + def _sanity_check(fast_tokenizer, slow_tokenizer, keep_fast_tokenizer=False): + """Set tokenizer(s) to `None` if the fast/slow tokenizers have different values for `vocab_size` or `length`. + + If `keep_fast_tokenizer=True`, the fast tokenizer will be kept. + """ + # sanity check 1: fast and slow tokenizers should be compatible (vocab_size) + if fast_tokenizer is not None and slow_tokenizer is not None: + if fast_tokenizer.vocab_size != slow_tokenizer.vocab_size: + warning_messagae = ( + "The fast/slow tokenizers " + f"({fast_tokenizer.__class__.__name__}/{slow_tokenizer.__class__.__name__}) have different " + "vocabulary size: " + f"fast_tokenizer.vocab_size = {fast_tokenizer.vocab_size} and " + f"slow_tokenizer.vocab_size = {slow_tokenizer.vocab_size}." + ) + result["warnings"].append(warning_messagae) + if not keep_fast_tokenizer: + fast_tokenizer = None + slow_tokenizer = None + + # sanity check 2: fast and slow tokenizers should be compatible (length) + if fast_tokenizer is not None and slow_tokenizer is not None: + if len(fast_tokenizer) != len(slow_tokenizer): + warning_messagae = ( + f"The fast/slow tokenizers () have different length: " + f"len(fast_tokenizer) = {len(fast_tokenizer)} and " + f"len(slow_tokenizer) = {len(slow_tokenizer)}." + ) + result["warnings"].append(warning_messagae) + if not keep_fast_tokenizer: + fast_tokenizer = None + slow_tokenizer = None + + return fast_tokenizer, slow_tokenizer + tokenizers = [] feature_extractors = [] for processor in processors: if isinstance(processor, PreTrainedTokenizerBase): - tokenizers.append(processor) + if processor.__class__.__name__ not in {x.__class__.__name__ for x in tokenizers}: + tokenizers.append(processor) elif isinstance(processor, BaseImageProcessor): - feature_extractors.append(processor) + if processor.__class__.__name__ not in {x.__class__.__name__ for x in feature_extractors}: + feature_extractors.append(processor) elif isinstance(processor, FeatureExtractionMixin): - feature_extractors.append(processor) + if processor.__class__.__name__ not in {x.__class__.__name__ for x in feature_extractors}: + feature_extractors.append(processor) elif isinstance(processor, ProcessorMixin): + if hasattr(processor, "tokenizer"): + if processor.tokenizer.__class__.__name__ not in {x.__class__.__name__ for x in tokenizers}: + tokenizers.append(processor.tokenizer) # Currently, we only have these 2 possibilities - tokenizers.append(processor.tokenizer) if hasattr(processor, "image_processor"): - feature_extractors.append(processor.image_processor) + if processor.image_processor.__class__.__name__ not in { + x.__class__.__name__ for x in feature_extractors + }: + feature_extractors.append(processor.image_processor) elif hasattr(processor, "feature_extractor"): - feature_extractors.append(processor.feature_extractor) + if processor.feature_extractor.__class__.__name__ not in { + x.__class__.__name__ for x in feature_extractors + }: + feature_extractors.append(processor.feature_extractor) # check the built processors have the unique type num_types = len({x.__class__.__name__ for x in feature_extractors}) @@ -511,65 +561,109 @@ def convert_processors(processors, tiny_config, output_folder, result): fast_tokenizer = None slow_tokenizer = None + for tokenizer in tokenizers: if isinstance(tokenizer, PreTrainedTokenizerFast): - if fast_tokenizer is None: - fast_tokenizer = tokenizer - try: - # Wav2Vec2ForCTC , ByT5Tokenizer etc. all are already small enough and have no fast version that can - # be retrained - if fast_tokenizer.vocab_size > TARGET_VOCAB_SIZE: - fast_tokenizer = convert_tokenizer(tokenizer) - except Exception: - result["warnings"].append( - ( - f"Failed to convert the fast tokenizer for {fast_tokenizer.__class__.__name__}.", - traceback.format_exc(), - ) - ) - continue - elif slow_tokenizer is None: + fast_tokenizer = tokenizer + else: slow_tokenizer = tokenizer - # Make sure the fast tokenizer can be saved + # If the (original) fast/slow tokenizers don't correspond, keep only the fast tokenizer. + # This doesn't necessarily imply the fast/slow tokenizers in a single Hub repo. has issues. + # It's more of an issue in `build_processor` which tries to get a checkpoint with as much effort as possible. + # For `YosoModel` (which uses `AlbertTokenizer(Fast)`), its real (Hub) checkpoint doesn't contain valid files to + # load the slower tokenizer (`AlbertTokenizer`), and it ends up finding the (canonical) checkpoint of `AlbertModel`, + # which has different vocabulary. + # TODO: Try to improve `build_processor`'s definition and/or usage to avoid the above situation in the first place. + fast_tokenizer, slow_tokenizer = _sanity_check(fast_tokenizer, slow_tokenizer, keep_fast_tokenizer=True) + original_fast_tokenizer, original_slow_tokenizer = fast_tokenizer, slow_tokenizer + if fast_tokenizer: try: - fast_tokenizer.save_pretrained(output_folder) + # Wav2Vec2ForCTC , ByT5Tokenizer etc. all are already small enough and have no fast version that can + # be retrained + if fast_tokenizer.vocab_size > TARGET_VOCAB_SIZE: + fast_tokenizer = convert_tokenizer(fast_tokenizer) except Exception: result["warnings"].append( ( - f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.", + f"Failed to convert the fast tokenizer for {fast_tokenizer.__class__.__name__}.", traceback.format_exc(), ) ) - fast_tokenizer = None - # Make sure the slow tokenizer (if any) corresponds to the fast version (as it might be converted above) + # If `fast_tokenizer` exists, `slow_tokenizer` should correspond to it. if fast_tokenizer: + # Make sure the fast tokenizer can be saved try: - slow_tokenizer = AutoTokenizer.from_pretrained(output_folder, use_fast=False) + # We don't save it to `output_folder` at this moment - only at the end of this function. + with tempfile.TemporaryDirectory() as tmpdir: + fast_tokenizer.save_pretrained(tmpdir) + try: + slow_tokenizer = AutoTokenizer.from_pretrained(tmpdir, use_fast=False) + except Exception: + result["warnings"].append( + ( + f"Failed to load the slow tokenizer saved from {fast_tokenizer.__class__.__name__}.", + traceback.format_exc(), + ) + ) + # Let's just keep the fast version + slow_tokenizer = None except Exception: result["warnings"].append( ( - f"Failed to load the slow tokenizer saved from {fast_tokenizer.__class__.__name__}.", + f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.", traceback.format_exc(), ) ) - # Let's just keep the fast version - slow_tokenizer = None + fast_tokenizer = None - # If the fast version can't be created and saved, let's use the slow version - if not fast_tokenizer and slow_tokenizer: - try: - slow_tokenizer.save_pretrained(output_folder) - except Exception: - result["warnings"].append( - ( - f"Failed to save the slow tokenizer for {slow_tokenizer.__class__.__name__}.", - traceback.format_exc(), + # If the (possibly converted) fast/slow tokenizers don't correspond, set them to `None`, and use the original + # tokenizers. + fast_tokenizer, slow_tokenizer = _sanity_check(fast_tokenizer, slow_tokenizer, keep_fast_tokenizer=False) + + # If there is any conversion failed, we keep the original tokenizers. + if (original_fast_tokenizer is not None and fast_tokenizer is None) or ( + original_slow_tokenizer is not None and slow_tokenizer is None + ): + warning_messagae = ( + "There are some issues when converting the fast/slow tokenizers. The original tokenizers from the Hub " + " will be used instead." + ) + result["warnings"].append(warning_messagae) + # Let's use the original version at the end (`original_fast_tokenizer` and `original_slow_tokenizer`) + fast_tokenizer = original_fast_tokenizer + slow_tokenizer = original_slow_tokenizer + + # Make sure the fast tokenizer can be saved + if fast_tokenizer: + # We don't save it to `output_folder` at this moment - only at the end of this function. + with tempfile.TemporaryDirectory() as tmpdir: + try: + fast_tokenizer.save_pretrained(tmpdir) + except Exception: + result["warnings"].append( + ( + f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.", + traceback.format_exc(), + ) ) - ) - slow_tokenizer = None + fast_tokenizer = None + # Make sure the slow tokenizer can be saved + if slow_tokenizer: + # We don't save it to `output_folder` at this moment - only at the end of this function. + with tempfile.TemporaryDirectory() as tmpdir: + try: + slow_tokenizer.save_pretrained(tmpdir) + except Exception: + result["warnings"].append( + ( + f"Failed to save the slow tokenizer for {slow_tokenizer.__class__.__name__}.", + traceback.format_exc(), + ) + ) + slow_tokenizer = None # update feature extractors using the tiny config try: @@ -883,7 +977,9 @@ def get_config_overrides(config_class, processors): return config_overrides # Get some properties of the (already converted) tokenizer (smaller vocab size, special token ids, etc.) - vocab_size = tokenizer.vocab_size + # We use `len(tokenizer)` instead of `tokenizer.vocab_size` to avoid potential issues for tokenizers with non-empty + # `added_tokens_encoder`. One example is the `DebertaV2Tokenizer` where the mask token is the extra token. + vocab_size = len(tokenizer) config_overrides["vocab_size"] = vocab_size # Used to create a new model tester with `tokenizer.vocab_size` in order to get the (updated) special token ids. From 9ac11ec3c7657d45a71dde2aab50a378eaac06ec Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 29 Mar 2023 10:59:42 -0400 Subject: [PATCH 220/277] Revert "Error (also in original) model, scaling only q matrix not qk.T dot product (qk.T/sqrt(dim_per_head))" (#22444) Revert "Error (also in original) model, scaling only q matrix not qk.T dot product (qk.T/sqrt(dim_per_head)) (#21627)" This reverts commit bad83008377bf01a34ac2e08c74e7da89eaf4e07. --- src/transformers/models/flaubert/modeling_flaubert.py | 3 ++- src/transformers/models/xlm/modeling_xlm.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/flaubert/modeling_flaubert.py b/src/transformers/models/flaubert/modeling_flaubert.py index 999f19795f84..bb7790730115 100644 --- a/src/transformers/models/flaubert/modeling_flaubert.py +++ b/src/transformers/models/flaubert/modeling_flaubert.py @@ -172,7 +172,8 @@ def unshape(x): k, v = cache[self.layer_id] cache[self.layer_id] = (k, v) - scores = torch.matmul(q, k.transpose(2, 3)) / math.sqrt(dim_per_head) # (bs, n_heads, qlen, klen) + q = q / math.sqrt(dim_per_head) # (bs, n_heads, qlen, dim_per_head) + scores = torch.matmul(q, k.transpose(2, 3)) # (bs, n_heads, qlen, klen) mask = (mask == 0).view(mask_reshape).expand_as(scores) # (bs, n_heads, qlen, klen) scores.masked_fill_(mask, torch.finfo(scores.dtype).min) # (bs, n_heads, qlen, klen) diff --git a/src/transformers/models/xlm/modeling_xlm.py b/src/transformers/models/xlm/modeling_xlm.py index 315206a19647..2d0adcddf95c 100755 --- a/src/transformers/models/xlm/modeling_xlm.py +++ b/src/transformers/models/xlm/modeling_xlm.py @@ -176,7 +176,8 @@ def unshape(x): k, v = cache[self.layer_id] cache[self.layer_id] = (k, v) - scores = torch.matmul(q, k.transpose(2, 3)) / math.sqrt(dim_per_head) # (bs, n_heads, qlen, klen) + q = q / math.sqrt(dim_per_head) # (bs, n_heads, qlen, dim_per_head) + scores = torch.matmul(q, k.transpose(2, 3)) # (bs, n_heads, qlen, klen) mask = (mask == 0).view(mask_reshape).expand_as(scores) # (bs, n_heads, qlen, klen) scores.masked_fill_(mask, torch.finfo(scores.dtype).min) # (bs, n_heads, qlen, klen) From deb36f085cd8848e9654353dc1f5535599aa143c Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:40:45 +0200 Subject: [PATCH 221/277] [`Pix2Struct`] Fix slow test (#22448) fix slow test --- tests/models/pix2struct/test_modeling_pix2struct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/pix2struct/test_modeling_pix2struct.py b/tests/models/pix2struct/test_modeling_pix2struct.py index dc219bbd61ab..f56f8f6d3ecf 100644 --- a/tests/models/pix2struct/test_modeling_pix2struct.py +++ b/tests/models/pix2struct/test_modeling_pix2struct.py @@ -406,7 +406,7 @@ def setUp(self): def test_model(self): config, input_dict = self.model_tester.prepare_config_and_inputs_for_common() for model_class in self.all_model_classes: - model = model_class(config) + model = model_class(config).to(torch_device) output = model(**input_dict) self.assertEqual( From e8320631e117d3867da5bd7e5198f521d2324605 Mon Sep 17 00:00:00 2001 From: jeffhataws <56947987+jeffhataws@users.noreply.github.com> Date: Wed, 29 Mar 2023 09:59:13 -0700 Subject: [PATCH 222/277] Revert "Fix --bf16 option support for Neuron after PR #22300" (#22451) This reverts commit fd81746dbec5f17c8285a0fdc72ca4b4c025cc33. --- src/transformers/trainer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 7267d79b3c86..a41d43edeb4f 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -588,12 +588,7 @@ def __init__( if args.fp16 or args.bf16: if args.half_precision_backend == "auto": - if is_torch_neuroncore_available(): - if args.fp16: - raise ValueError("Tried to use `fp16` but this option is not yet supported on Neuron.") - else: - args.half_precision_backend = "cpu_amp" - elif args.device == torch.device("cpu"): + if args.device == torch.device("cpu"): if args.fp16: raise ValueError("Tried to use `fp16` but it is not supported on cpu") elif _is_native_cpu_amp_available: From 78293e25a761de96219c27b673fc90d6b47ef480 Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 29 Mar 2023 20:15:38 +0300 Subject: [PATCH 223/277] Update Neptune docs (#22452) --- examples/pytorch/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/pytorch/README.md b/examples/pytorch/README.md index aa669932475c..60d762c69e10 100644 --- a/examples/pytorch/README.md +++ b/examples/pytorch/README.md @@ -262,13 +262,13 @@ First, install the Neptune client library. You can do it with either `pip` or `c `pip`: ```bash -pip install neptune-client +pip install neptune ``` `conda`: ```bash -conda install -c conda-forge neptune-client +conda install -c conda-forge neptune ``` Next, in your model training script, import `NeptuneCallback`: @@ -283,8 +283,8 @@ To enable Neptune logging, in your `TrainingArguments`, set the `report_to` argu training_args = TrainingArguments( "quick-training-distilbert-mrpc", evaluation_strategy="steps", - eval_steps = 20, - report_to = "neptune", + eval_steps=20, + report_to="neptune", ) trainer = Trainer( @@ -294,6 +294,8 @@ trainer = Trainer( ) ``` +**Note:** This method requires saving your Neptune credentials as environment variables (see the bottom of the section). + Alternatively, for more logging options, create a Neptune callback: ```python @@ -318,7 +320,7 @@ neptune_callback = NeptuneCallback( Pass the callback to the Trainer: ```python -training_args = TrainingArguments(..., report_to = None) +training_args = TrainingArguments(..., report_to=None) trainer = Trainer( model, training_args, @@ -336,7 +338,7 @@ Now, when you start the training with `trainer.train()`, your metadata will be l | `NEPTUNE_API_TOKEN` | Your Neptune API token. To find and copy it, click your Neptune avatar and select **Get your API token**. | | `NEPTUNE_PROJECT` | The full name of your Neptune project (`workspace-name/project-name`). To find and copy it, head to **project settings** → **Properties**. | -For detailed instructions and examples, see the [Neptune docs](https://docs.neptune.ai/integrations-and-supported-tools/model-training/hugging-face). +For detailed instructions and examples, see the [Neptune docs](https://docs.neptune.ai/integrations/transformers/). ### ClearML @@ -373,4 +375,4 @@ Advanced configuration is possible by setting environment variables: | CLEARML_PROJECT | Name of the project in ClearML. (default: `"HuggingFace Transformers"`) | | CLEARML_TASK | Name of the task in ClearML. (default: `"Trainer"`) | -Additional configuration options are available through generic [clearml environment variables](https://clear.ml/docs/latest/docs/configs/env_vars). \ No newline at end of file +Additional configuration options are available through generic [clearml environment variables](https://clear.ml/docs/latest/docs/configs/env_vars). From 85cbc37d3cae61b6a59c1652507a8fbed2ae0a6d Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 29 Mar 2023 19:45:06 +0200 Subject: [PATCH 224/277] Avoid using personal HF token in CI (#22453) Co-authored-by: ydshieh --- .github/workflows/update_tiny_models.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_tiny_models.yml b/.github/workflows/update_tiny_models.yml index 1ed8271f8ab2..bf70c4f93729 100644 --- a/.github/workflows/update_tiny_models.yml +++ b/.github/workflows/update_tiny_models.yml @@ -9,7 +9,7 @@ on: - cron: "0 2 * * *" env: - TOKEN: ${{ secrets.SYLVAIN_HF_TOKEN }} + TOKEN: ${{ secrets.TRANSFORMERS_HUB_BOT_HF_TOKEN }} jobs: update_tiny_models: From b35dcee6c47e05ebbe7a7b4a3e9fffa214ad409f Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:05:42 -0400 Subject: [PATCH 225/277] Update release instructions (#22454) --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 33eefac65746..89a99a4d9dfb 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,10 @@ 7. Build both the sources and the wheel. Do not change anything in setup.py between creating the wheel and the source distribution (obviously). + Clean up your build and dist folders (to avoid re-uploading oldies): + rm -rf dist + rm -rf build + For the wheel, run: "python setup.py bdist_wheel" in the top level directory. (this will build a wheel for the python version you use to build it). @@ -46,10 +50,10 @@ 8. Check that everything looks correct by uploading the package to the pypi test server: - twine upload dist/* -r pypitest + twine upload dist/* -r testpypi (pypi suggest using twine as other methods upload files via plaintext.) You may have to specify the repository url, use the following command then: - twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/ + twine upload dist/* -r testpypi --repository-url=https://test.pypi.org/legacy/ Check that you can install it in a virtualenv by running: pip install -i https://testpypi.python.org/pypi transformers @@ -58,6 +62,8 @@ python -c "from transformers import pipeline; classifier = pipeline('text-classification'); print(classifier('What a nice release'))" python -c "from transformers import *" + If making a patch release, double check the bug you are patching is indeed resolved. + 9. Upload the final version to actual pypi: twine upload dist/* -r pypi From 00dd0d13b4a2b48ff20b256eac9e1efdacb73e19 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:07:06 -0400 Subject: [PATCH 226/277] Pin ruff (#22455) --- setup.py | 2 +- src/transformers/dependency_versions_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 89a99a4d9dfb..ce609f240483 100644 --- a/setup.py +++ b/setup.py @@ -159,7 +159,7 @@ "rhoknp>=1.1.0", "rjieba", "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1", - "ruff>=0.0.241", + "ruff>=0.0.241,<=0.0.259", "sacrebleu>=1.4.12,<2.0.0", "sacremoses", "safetensors>=0.2.1", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 314267922b1a..9c8b14711eb3 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -59,7 +59,7 @@ "rhoknp": "rhoknp>=1.1.0", "rjieba": "rjieba", "rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1", - "ruff": "ruff>=0.0.241", + "ruff": "ruff>=0.0.241,<=0.0.259", "sacrebleu": "sacrebleu>=1.4.12,<2.0.0", "sacremoses": "sacremoses", "safetensors": "safetensors>=0.2.1", From 720a2900f06985e339bf4a9800d609a1ffb7fd15 Mon Sep 17 00:00:00 2001 From: Stefan Heng <43276957+StefanHeng@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:36:39 -0400 Subject: [PATCH 227/277] Update: ignore padding support for TransfoXL training when n_clusters==0 (#22457) * Update: ignore padding support for TransfoXL training when n_clusters==0 * Update: transformer XL always pad * Update: drop doc --- .../models/transfo_xl/modeling_transfo_xl_utilities.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py b/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py index e25ba2cd476a..addf2a08372b 100644 --- a/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py +++ b/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py @@ -86,7 +86,7 @@ def forward(self, hidden, labels=None, keep_order=False): """ Params: hidden :: [len*bsz x d_proj] - labels :: [len*bsz + labels :: [len*bsz] Return: if labels is None: out :: [len*bsz x n_tokens] log probabilities of tokens over the vocabulary else: out :: @@ -109,7 +109,11 @@ def forward(self, hidden, labels=None, keep_order=False): if self.n_clusters == 0: logit = self._compute_logit(hidden, self.out_layers[0].weight, self.out_layers[0].bias, self.out_projs[0]) if labels is not None: - out = -nn.functional.log_softmax(logit, dim=-1).gather(1, labels.unsqueeze(1)).squeeze(1) + mask = labels != -100 + out = torch.zeros_like(labels, dtype=hidden.dtype, device=hidden.device) + out[mask] = ( + -nn.functional.log_softmax(logit, dim=-1)[mask].gather(1, labels[mask].unsqueeze(1)).squeeze(1) + ) else: out = nn.functional.log_softmax(logit, dim=-1) else: From 817428d27086cdfb2896a71cad895bc05e5ac1bc Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Thu, 30 Mar 2023 10:04:11 +0100 Subject: [PATCH 228/277] Move common properties to BackboneMixin (#21855) * Move common properties to BackboneMixin * Fix failing tests * Update ConvNextV2 backbone --- src/transformers/modeling_utils.py | 20 ++++++++++++++++++- src/transformers/models/bit/modeling_bit.py | 12 +---------- .../models/convnext/modeling_convnext.py | 12 +---------- .../models/convnextv2/modeling_convnextv2.py | 12 +---------- .../models/dinat/modeling_dinat.py | 11 +--------- .../maskformer/modeling_maskformer_swin.py | 10 +--------- src/transformers/models/nat/modeling_nat.py | 11 +--------- .../models/resnet/modeling_resnet.py | 12 +---------- src/transformers/models/swin/modeling_swin.py | 11 +--------- 9 files changed, 27 insertions(+), 84 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 913d7e4a9f1c..384876fb6de2 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -968,12 +968,30 @@ def floating_point_ops( class BackboneMixin: + @property + def out_feature_channels(self): + # the current backbones will output the number of channels for each stage + # even if that stage is not in the out_features list. + return {stage: self.num_features[i] for i, stage in enumerate(self.stage_names)} + + @property + def channels(self): + return [self.out_feature_channels[name] for name in self.out_features] + def forward_with_filtered_kwargs(self, *args, **kwargs): signature = dict(inspect.signature(self.forward).parameters) filtered_kwargs = {k: v for k, v in kwargs.items() if k in signature} - return self(*args, **filtered_kwargs) + def forward( + self, + pixel_values: Tensor, + output_hidden_states: Optional[bool] = None, + output_attentions: Optional[bool] = None, + return_dict: Optional[bool] = None, + ): + raise NotImplementedError("This method should be implemented by the derived class.") + class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMixin): r""" diff --git a/src/transformers/models/bit/modeling_bit.py b/src/transformers/models/bit/modeling_bit.py index 7ebe461e5be0..b149bf412a76 100644 --- a/src/transformers/models/bit/modeling_bit.py +++ b/src/transformers/models/bit/modeling_bit.py @@ -849,21 +849,11 @@ def __init__(self, config): self.bit = BitModel(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] - - out_feature_channels = {} - out_feature_channels["stem"] = config.embedding_size - for idx, stage in enumerate(self.stage_names[1:]): - out_feature_channels[stage] = config.hidden_sizes[idx] - - self.out_feature_channels = out_feature_channels + self.num_features = [config.embedding_size] + config.hidden_sizes # initialize weights and apply final processing self.post_init() - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - @add_start_docstrings_to_model_forward(BIT_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) def forward( diff --git a/src/transformers/models/convnext/modeling_convnext.py b/src/transformers/models/convnext/modeling_convnext.py index 3ba8062b77c0..7076e64ccbe9 100755 --- a/src/transformers/models/convnext/modeling_convnext.py +++ b/src/transformers/models/convnext/modeling_convnext.py @@ -486,13 +486,7 @@ def __init__(self, config): self.encoder = ConvNextEncoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] - - out_feature_channels = {} - out_feature_channels["stem"] = config.hidden_sizes[0] - for idx, stage in enumerate(self.stage_names[1:]): - out_feature_channels[stage] = config.hidden_sizes[idx] - - self.out_feature_channels = out_feature_channels + self.num_features = [config.hidden_sizes[0]] + config.hidden_sizes # Add layer norms to hidden states of out_features hidden_states_norms = {} @@ -503,10 +497,6 @@ def __init__(self, config): # initialize weights and apply final processing self.post_init() - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - @add_start_docstrings_to_model_forward(CONVNEXT_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) def forward( diff --git a/src/transformers/models/convnextv2/modeling_convnextv2.py b/src/transformers/models/convnextv2/modeling_convnextv2.py index 3ee1b34f385e..741049af8e98 100644 --- a/src/transformers/models/convnextv2/modeling_convnextv2.py +++ b/src/transformers/models/convnextv2/modeling_convnextv2.py @@ -509,13 +509,7 @@ def __init__(self, config): self.encoder = ConvNextV2Encoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] - - out_feature_channels = {} - out_feature_channels["stem"] = config.hidden_sizes[0] - for idx, stage in enumerate(self.stage_names[1:]): - out_feature_channels[stage] = config.hidden_sizes[idx] - - self.out_feature_channels = out_feature_channels + self.num_features = [config.hidden_sizes[0]] + config.hidden_sizes # Add layer norms to hidden states of out_features hidden_states_norms = {} @@ -526,10 +520,6 @@ def __init__(self, config): # initialize weights and apply final processing self.post_init() - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - @add_start_docstrings_to_model_forward(CONVNEXTV2_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) def forward( diff --git a/src/transformers/models/dinat/modeling_dinat.py b/src/transformers/models/dinat/modeling_dinat.py index 2cacab8ac935..3e102d1052dd 100644 --- a/src/transformers/models/dinat/modeling_dinat.py +++ b/src/transformers/models/dinat/modeling_dinat.py @@ -891,12 +891,7 @@ def __init__(self, config): self.encoder = DinatEncoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] - - num_features = [int(config.embed_dim * 2**i) for i in range(len(config.depths))] - self.out_feature_channels = {} - self.out_feature_channels["stem"] = config.embed_dim - for i, stage in enumerate(self.stage_names[1:]): - self.out_feature_channels[stage] = num_features[i] + self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] # Add layer norms to hidden states of out_features hidden_states_norms = {} @@ -910,10 +905,6 @@ def __init__(self, config): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - @add_start_docstrings_to_model_forward(DINAT_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) def forward( diff --git a/src/transformers/models/maskformer/modeling_maskformer_swin.py b/src/transformers/models/maskformer/modeling_maskformer_swin.py index 91a61712ffbe..e84f376c045e 100644 --- a/src/transformers/models/maskformer/modeling_maskformer_swin.py +++ b/src/transformers/models/maskformer/modeling_maskformer_swin.py @@ -859,20 +859,12 @@ def __init__(self, config: MaskFormerSwinConfig): if "stem" in self.out_features: raise ValueError("This backbone does not support 'stem' in the `out_features`.") - num_features = [int(config.embed_dim * 2**i) for i in range(len(config.depths))] - self.out_feature_channels = {} - for i, stage in enumerate(self.stage_names[1:]): - self.out_feature_channels[stage] = num_features[i] - + self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] self.hidden_states_norms = nn.ModuleList([nn.LayerNorm(num_channels) for num_channels in self.channels]) # Initialize weights and apply final processing self.post_init() - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - def forward( self, pixel_values: Tensor, diff --git a/src/transformers/models/nat/modeling_nat.py b/src/transformers/models/nat/modeling_nat.py index dfe801d19961..fcc05947ed1b 100644 --- a/src/transformers/models/nat/modeling_nat.py +++ b/src/transformers/models/nat/modeling_nat.py @@ -869,12 +869,7 @@ def __init__(self, config): self.encoder = NatEncoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] - - num_features = [int(config.embed_dim * 2**i) for i in range(len(config.depths))] - self.out_feature_channels = {} - self.out_feature_channels["stem"] = config.embed_dim - for i, stage in enumerate(self.stage_names[1:]): - self.out_feature_channels[stage] = num_features[i] + self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] # Add layer norms to hidden states of out_features hidden_states_norms = {} @@ -888,10 +883,6 @@ def __init__(self, config): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - @add_start_docstrings_to_model_forward(NAT_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) def forward( diff --git a/src/transformers/models/resnet/modeling_resnet.py b/src/transformers/models/resnet/modeling_resnet.py index ce11c3d32989..946d6b9a0ece 100644 --- a/src/transformers/models/resnet/modeling_resnet.py +++ b/src/transformers/models/resnet/modeling_resnet.py @@ -437,21 +437,11 @@ def __init__(self, config): self.encoder = ResNetEncoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] - - out_feature_channels = {} - out_feature_channels["stem"] = config.embedding_size - for idx, stage in enumerate(self.stage_names[1:]): - out_feature_channels[stage] = config.hidden_sizes[idx] - - self.out_feature_channels = out_feature_channels + self.num_features = [config.embedding_size] + config.hidden_sizes # initialize weights and apply final processing self.post_init() - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - @add_start_docstrings_to_model_forward(RESNET_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=BackboneOutput, config_class=_CONFIG_FOR_DOC) def forward( diff --git a/src/transformers/models/swin/modeling_swin.py b/src/transformers/models/swin/modeling_swin.py index 17cd0a3d1ad2..8c1c7a0b655f 100644 --- a/src/transformers/models/swin/modeling_swin.py +++ b/src/transformers/models/swin/modeling_swin.py @@ -1255,12 +1255,7 @@ def __init__(self, config: SwinConfig): self.encoder = SwinEncoder(config, self.embeddings.patch_grid) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] - - num_features = [int(config.embed_dim * 2**i) for i in range(len(config.depths))] - self.out_feature_channels = {} - self.out_feature_channels["stem"] = config.embed_dim - for i, stage in enumerate(self.stage_names[1:]): - self.out_feature_channels[stage] = num_features[i] + self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] # Add layer norms to hidden states of out_features hidden_states_norms = {} @@ -1274,10 +1269,6 @@ def __init__(self, config: SwinConfig): def get_input_embeddings(self): return self.embeddings.patch_embeddings - @property - def channels(self): - return [self.out_feature_channels[name] for name in self.out_features] - def forward( self, pixel_values: torch.Tensor, From 11d5ddf7e2fe71f7c10616370ef9230f71a01c25 Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Thu, 30 Mar 2023 11:29:11 +0100 Subject: [PATCH 229/277] Rescale image back if it was scaled during PIL conversion (#22458) * Rescale image back if it was scaled during PIL conversion * do_rescale is defined if PIL image passed in --- src/transformers/image_transforms.py | 53 ++++++++++++++++++---------- tests/test_image_transforms.py | 8 +++++ 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/transformers/image_transforms.py b/src/transformers/image_transforms.py index 8f3fac73dd5f..369ddc8d4c00 100644 --- a/src/transformers/image_transforms.py +++ b/src/transformers/image_transforms.py @@ -118,6 +118,33 @@ def rescale( return rescaled_image +def _rescale_for_pil_conversion(image): + """ + Detects whether or not the image needs to be rescaled before being converted to a PIL image. + + The assumption is that if the image is of type `np.float` and all values are between 0 and 1, it needs to be + rescaled. + """ + if image.dtype == np.uint8: + do_rescale = False + elif np.allclose(image, image.astype(int)): + if np.all(0 <= image) and np.all(image <= 255): + do_rescale = False + else: + raise ValueError( + "The image to be converted to a PIL image contains values outside the range [0, 255], " + f"got [{image.min()}, {image.max()}] which cannot be converted to uint8." + ) + elif np.all(0 <= image) and np.all(image <= 1): + do_rescale = True + else: + raise ValueError( + "The image to be converted to a PIL image contains values outside the range [0, 1], " + f"got [{image.min()}, {image.max()}] which cannot be converted to uint8." + ) + return do_rescale + + def to_pil_image( image: Union[np.ndarray, "PIL.Image.Image", "torch.Tensor", "tf.Tensor", "jnp.ndarray"], do_rescale: Optional[bool] = None, @@ -157,24 +184,7 @@ def to_pil_image( image = np.squeeze(image, axis=-1) if image.shape[-1] == 1 else image # PIL.Image can only store uint8 values so we rescale the image to be between 0 and 255 if needed. - if do_rescale is None: - if image.dtype == np.uint8: - do_rescale = False - elif np.allclose(image, image.astype(int)): - if np.all(0 <= image) and np.all(image <= 255): - do_rescale = False - else: - raise ValueError( - "The image to be converted to a PIL image contains values outside the range [0, 255], " - f"got [{image.min()}, {image.max()}] which cannot be converted to uint8." - ) - elif np.all(0 <= image) and np.all(image <= 1): - do_rescale = True - else: - raise ValueError( - "The image to be converted to a PIL image contains values outside the range [0, 1], " - f"got [{image.min()}, {image.max()}] which cannot be converted to uint8." - ) + do_rescale = _rescale_for_pil_conversion(image) if do_rescale is None else do_rescale if do_rescale: image = rescale(image, 255) @@ -291,8 +301,10 @@ def resize( # To maintain backwards compatibility with the resizing done in previous image feature extractors, we use # the pillow library to resize the image and then convert back to numpy + do_rescale = False if not isinstance(image, PIL.Image.Image): - image = to_pil_image(image) + do_rescale = _rescale_for_pil_conversion(image) + image = to_pil_image(image, do_rescale=do_rescale) height, width = size # PIL images are in the format (width, height) resized_image = image.resize((width, height), resample=resample, reducing_gap=reducing_gap) @@ -306,6 +318,9 @@ def resize( resized_image = to_channel_dimension_format( resized_image, data_format, input_channel_dim=ChannelDimension.LAST ) + # If an image was rescaled to be in the range [0, 255] before converting to a PIL image, then we need to + # rescale it back to the original range. + resized_image = rescale(resized_image, 1 / 255) if do_rescale else resized_image return resized_image diff --git a/tests/test_image_transforms.py b/tests/test_image_transforms.py index 79580e0876e4..cb1524ac12cf 100644 --- a/tests/test_image_transforms.py +++ b/tests/test_image_transforms.py @@ -249,6 +249,14 @@ def test_resize(self): # PIL size is in (width, height) order self.assertEqual(resized_image.size, (40, 30)) + # Check an image with float values between 0-1 is returned with values in this range + image = np.random.rand(3, 224, 224) + resized_image = resize(image, (30, 40)) + self.assertIsInstance(resized_image, np.ndarray) + self.assertEqual(resized_image.shape, (3, 30, 40)) + self.assertTrue(np.all(resized_image >= 0)) + self.assertTrue(np.all(resized_image <= 1)) + def test_normalize(self): image = np.random.randint(0, 256, (224, 224, 3)) / 255 From ccee6124e5c021df45e31e47506b3aad6d7156d7 Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Thu, 30 Mar 2023 11:30:19 +0100 Subject: [PATCH 230/277] Skip flaky NLLB Moe test for now (#22463) Skip flaky test for now --- tests/models/nllb_moe/test_modeling_nllb_moe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/models/nllb_moe/test_modeling_nllb_moe.py b/tests/models/nllb_moe/test_modeling_nllb_moe.py index e2fb1483f7f8..567aab56eaea 100644 --- a/tests/models/nllb_moe/test_modeling_nllb_moe.py +++ b/tests/models/nllb_moe/test_modeling_nllb_moe.py @@ -21,6 +21,7 @@ from transformers import NllbMoeConfig, is_torch_available, set_seed from transformers.testing_utils import ( + is_flaky, require_sentencepiece, require_tokenizers, require_torch, @@ -281,6 +282,7 @@ def test_save_load_strict(self): model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True) self.assertEqual(info["missing_keys"], []) + @is_flaky() def test_decoder_model_past_with_large_inputs(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs) From 8e0179a4ce2f0f5141c175fb924135e8a43574c2 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Thu, 30 Mar 2023 12:00:12 +0100 Subject: [PATCH 231/277] Generate: basic token streaming (#22449) * haha tokens go brrrr --- docs/source/en/generation_strategies.mdx | 23 ++++ docs/source/en/internal/generation_utils.mdx | 4 + .../en/main_classes/text_generation.mdx | 3 +- src/transformers/__init__.py | 4 +- src/transformers/generation/__init__.py | 4 +- src/transformers/generation/streamers.py | 104 ++++++++++++++++++ src/transformers/generation/utils.py | 51 ++++++++- tests/generation/test_streamers.py | 44 ++++++++ 8 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 src/transformers/generation/streamers.py create mode 100644 tests/generation/test_streamers.py diff --git a/docs/source/en/generation_strategies.mdx b/docs/source/en/generation_strategies.mdx index 44692169405f..831c8772b6c6 100644 --- a/docs/source/en/generation_strategies.mdx +++ b/docs/source/en/generation_strategies.mdx @@ -139,6 +139,29 @@ one for summarization with beam search). You must have the right Hub permissions ['Les fichiers de configuration sont faciles à utiliser !'] ``` +## Streaming + +The `generate()` supports streaming, through its `streamer` input. The `streamer` input is compatible any instance +from a class that has the following methods: `put()` and `end()`. Internally, `put()` is used to push new tokens and +`end()` is used to flag the end of text generation. + +In practice, you can craft your own streaming class for all sorts of purposes! We also have basic streaming classes +ready for you to use. For example, you can use the [`TextStreamer`] class to stream the output of `generate()` into +your screen, one word at a time: + +```python +>>> from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer + +>>> tok = AutoTokenizer.from_pretrained("gpt2") +>>> model = AutoModelForCausalLM.from_pretrained("gpt2") +>>> inputs = tok(["An increasing sequence: one,"], return_tensors="pt") +>>> streamer = TextStreamer(tok) + +>>> # Despite returning the usual output, the streamer will also print the generated text to stdout. +>>> _ = model.generate(**inputs, streamer=streamer, max_new_tokens=20) +An increasing sequence: one, two, three, four, five, six, seven, eight, nine, ten, eleven, +``` + ## Decoding strategies Certain combinations of the `generate()` parameters, and ultimately `generation_config`, can be used to enable specific diff --git a/docs/source/en/internal/generation_utils.mdx b/docs/source/en/internal/generation_utils.mdx index 3c86b7dc3f09..dd93c79e92bf 100644 --- a/docs/source/en/internal/generation_utils.mdx +++ b/docs/source/en/internal/generation_utils.mdx @@ -265,3 +265,7 @@ A [`Constraint`] can be used to force the generation to include specific tokens [[autodoc]] top_k_top_p_filtering [[autodoc]] tf_top_k_top_p_filtering + +## Streamers + +[[autodoc]] TextStreamer diff --git a/docs/source/en/main_classes/text_generation.mdx b/docs/source/en/main_classes/text_generation.mdx index 5351129cbb1d..39a15160346f 100644 --- a/docs/source/en/main_classes/text_generation.mdx +++ b/docs/source/en/main_classes/text_generation.mdx @@ -24,7 +24,8 @@ of the generation method. To learn how to inspect a model's generation configuration, what are the defaults, how to change the parameters ad hoc, and how to create and save a customized generation configuration, refer to the -[text generation strategies guide](../generation_strategies). +[text generation strategies guide](../generation_strategies). The guide also explains how to use related features, +like token streaming. ## GenerationConfig diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 52a770bc5c7e..518c6524d334 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -96,7 +96,7 @@ "feature_extraction_sequence_utils": ["SequenceFeatureExtractor"], "feature_extraction_utils": ["BatchFeature", "FeatureExtractionMixin"], "file_utils": [], - "generation": ["GenerationConfig"], + "generation": ["GenerationConfig", "TextStreamer"], "hf_argparser": ["HfArgumentParser"], "image_transforms": [], "integrations": [ @@ -3770,7 +3770,7 @@ from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin # Generation - from .generation import GenerationConfig + from .generation import GenerationConfig, TextStreamer from .hf_argparser import HfArgumentParser # Integrations diff --git a/src/transformers/generation/__init__.py b/src/transformers/generation/__init__.py index a5f4aa014918..d163c44dc7f0 100644 --- a/src/transformers/generation/__init__.py +++ b/src/transformers/generation/__init__.py @@ -17,8 +17,7 @@ from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available -_import_structure = {"configuration_utils": ["GenerationConfig"]} - +_import_structure = {"configuration_utils": ["GenerationConfig"], "streamers": ["TextStreamer"]} try: if not is_torch_available(): @@ -150,6 +149,7 @@ if TYPE_CHECKING: from .configuration_utils import GenerationConfig + from .streamers import TextStreamer try: if not is_torch_available(): diff --git a/src/transformers/generation/streamers.py b/src/transformers/generation/streamers.py new file mode 100644 index 000000000000..d110693b0eac --- /dev/null +++ b/src/transformers/generation/streamers.py @@ -0,0 +1,104 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from ..models.auto import AutoTokenizer + + +class BaseStreamer: + """ + Base class from which `.generate()` streamers should inherit. + """ + + def put(self, value): + """Function that is called by `.generate()` to push new tokens""" + raise NotImplementedError() + + def end(self): + """Function that is called by `.generate()` to signal the end of generation""" + raise NotImplementedError() + + +class TextStreamer(BaseStreamer): + """ + Simple text streamer that prints the token(s) to stdout as soon as entire words are formed. + + Parameters: + tokenizer (`AutoTokenizer`): + The tokenized used to decode the tokens. + + Examples: + + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer + + >>> tok = AutoTokenizer.from_pretrained("gpt2") + >>> model = AutoModelForCausalLM.from_pretrained("gpt2") + >>> inputs = tok(["An increasing sequence: one,"], return_tensors="pt") + >>> streamer = TextStreamer(tok) + + >>> # Despite returning the usual output, the streamer will also print the generated text to stdout. + >>> _ = model.generate(**inputs, streamer=streamer, max_new_tokens=20) + An increasing sequence: one, two, three, four, five, six, seven, eight, nine, ten, eleven, + ``` + """ + + def __init__(self, tokenizer: "AutoTokenizer"): + self.tokenizer = tokenizer + self.token_cache = [] + self.print_len = 0 + + def put(self, value): + """ + Recives tokens, decodes them, and prints them to stdout as soon as they form entire words. + """ + if len(value.shape) > 1 and value.shape[0] > 1: + raise ValueError("TextStreamer only supports batch size 1") + elif len(value.shape) > 1: + value = value[0] + + # Add the new token to the cache and decodes the entire thing. + self.token_cache.extend(value.tolist()) + text = self.tokenizer.decode(self.token_cache) + + # After symbol for a new line, we flush the cache. + if text.endswith("\n"): + printable_text = text[self.print_len :] + self.token_cache = [] + self.print_len = 0 + # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, + # which may change with the subsequent token -- there are probably smarter ways to do this!) + else: + printable_text = text[self.print_len : text.rfind(" ") + 1] + self.print_len += len(printable_text) + + print(printable_text, flush=True, end="") + + def end(self): + """Flushes any remaining cache and prints a newline to stdout.""" + # Flush the cache, if it exists + if len(self.token_cache) > 0: + text = self.tokenizer.decode(self.token_cache) + printable_text = text[self.print_len :] + self.token_cache = [] + self.print_len = 0 + else: + printable_text = "" + + # Print a newline (and the remaining text, if any) + print(printable_text, flush=True) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index ee92e51a19a5..ae12ae2930fc 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -18,7 +18,7 @@ import inspect import warnings from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union import torch import torch.distributed as dist @@ -72,6 +72,10 @@ ) +if TYPE_CHECKING: + from .streamers import BaseStreamer + + logger = logging.get_logger(__name__) @@ -1116,6 +1120,7 @@ def generate( stopping_criteria: Optional[StoppingCriteriaList] = None, prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None, synced_gpus: Optional[bool] = None, + streamer: Optional["BaseStreamer"] = None, **kwargs, ) -> Union[GenerateOutput, torch.LongTensor]: r""" @@ -1165,6 +1170,9 @@ def generate( Whether to continue running the while loop until max_length. Unless overridden this flag will be set to `True` under DeepSpeed ZeRO Stage 3 multiple GPUs environment to avoid hanging if one GPU finished generating before other GPUs. Otherwise it'll be set to `False`. + streamer (`BaseStreamer`, *optional*): + Streamer object that will be used to stream the generated sequences. Generated tokens are passed + through `streamer.put(token_ids)` and the streamer is responsible for any further processing. kwargs: Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be @@ -1295,6 +1303,9 @@ def generate( else: input_ids = inputs_tensor if model_input_name == "input_ids" else model_kwargs.pop("input_ids") + if streamer is not None: + streamer.put(input_ids.cpu()) + # 6. Prepare `max_length` depending on other stopping criteria. input_ids_seq_length = input_ids.shape[-1] has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None @@ -1335,7 +1346,8 @@ def generate( ) is_contrastive_search_gen_mode = ( - generation_config.top_k is not None + (generation_config.num_beams == 1) + and generation_config.top_k is not None and generation_config.top_k > 1 and generation_config.do_sample is False and generation_config.penalty_alpha is not None @@ -1384,6 +1396,11 @@ def generate( "Diverse beam search cannot be used in sampling mode. Make sure that `do_sample` is set to `False`." ) + if streamer is not None and (generation_config.num_beams > 1): + raise ValueError( + "`streamer` cannot be used with beam search (yet!). Make sure that `num_beams` is set to 1." + ) + if self.device.type != input_ids.device.type: warnings.warn( "You are calling .generate() with the `input_ids` being on a device type different" @@ -1426,6 +1443,7 @@ def generate( output_scores=generation_config.output_scores, return_dict_in_generate=generation_config.return_dict_in_generate, synced_gpus=synced_gpus, + streamer=streamer, **model_kwargs, ) @@ -1447,6 +1465,7 @@ def generate( output_scores=generation_config.output_scores, return_dict_in_generate=generation_config.return_dict_in_generate, synced_gpus=synced_gpus, + streamer=streamer, **model_kwargs, ) @@ -1473,6 +1492,7 @@ def generate( output_scores=generation_config.output_scores, return_dict_in_generate=generation_config.return_dict_in_generate, synced_gpus=synced_gpus, + streamer=streamer, **model_kwargs, ) @@ -1703,6 +1723,7 @@ def contrastive_search( output_scores: Optional[bool] = None, return_dict_in_generate: Optional[bool] = None, synced_gpus: Optional[bool] = False, + streamer: Optional["BaseStreamer"] = None, **model_kwargs, ) -> Union[ContrastiveSearchOutput, torch.LongTensor]: r""" @@ -1750,6 +1771,9 @@ def contrastive_search( Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. synced_gpus (`bool`, *optional*, defaults to `False`): Whether to continue running the while loop until max_length (needed for ZeRO stage 3) + streamer (`BaseStreamer`, *optional*): + Streamer object that will be used to stream the generated sequences. Generated tokens are passed + through `streamer.put(token_ids)` and the streamer is responsible for any further processing. model_kwargs: Additional model specific keyword arguments will be forwarded to the `forward` function of the model. If model is an encoder-decoder model the kwargs should include `encoder_outputs`. @@ -2010,6 +2034,8 @@ def contrastive_search( # update generated ids, model inputs, and length for next step input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + if streamer is not None: + streamer.put(next_tokens.cpu()) model_kwargs = self._update_model_kwargs_for_generation( outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder ) @@ -2027,6 +2053,9 @@ def contrastive_search( else: this_peer_finished = True + if streamer is not None: + streamer.end() + if return_dict_in_generate: if self.config.is_encoder_decoder: return ContrastiveSearchEncoderDecoderOutput( @@ -2061,6 +2090,7 @@ def greedy_search( output_scores: Optional[bool] = None, return_dict_in_generate: Optional[bool] = None, synced_gpus: Optional[bool] = False, + streamer: Optional["BaseStreamer"] = None, **model_kwargs, ) -> Union[GreedySearchOutput, torch.LongTensor]: r""" @@ -2105,6 +2135,9 @@ def greedy_search( Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. synced_gpus (`bool`, *optional*, defaults to `False`): Whether to continue running the while loop until max_length (needed for ZeRO stage 3) + streamer (`BaseStreamer`, *optional*): + Streamer object that will be used to stream the generated sequences. Generated tokens are passed + through `streamer.put(token_ids)` and the streamer is responsible for any further processing. model_kwargs: Additional model specific keyword arguments will be forwarded to the `forward` function of the model. If model is an encoder-decoder model the kwargs should include `encoder_outputs`. @@ -2256,6 +2289,8 @@ def greedy_search( # update generated ids, model inputs, and length for next step input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + if streamer is not None: + streamer.put(next_tokens.cpu()) model_kwargs = self._update_model_kwargs_for_generation( outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder ) @@ -2273,6 +2308,9 @@ def greedy_search( else: this_peer_finished = True + if streamer is not None: + streamer.end() + if return_dict_in_generate: if self.config.is_encoder_decoder: return GreedySearchEncoderDecoderOutput( @@ -2308,6 +2346,7 @@ def sample( output_scores: Optional[bool] = None, return_dict_in_generate: Optional[bool] = None, synced_gpus: Optional[bool] = False, + streamer: Optional["BaseStreamer"] = None, **model_kwargs, ) -> Union[SampleOutput, torch.LongTensor]: r""" @@ -2354,6 +2393,9 @@ def sample( Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. synced_gpus (`bool`, *optional*, defaults to `False`): Whether to continue running the while loop until max_length (needed for ZeRO stage 3) + streamer (`BaseStreamer`, *optional*): + Streamer object that will be used to stream the generated sequences. Generated tokens are passed + through `streamer.put(token_ids)` and the streamer is responsible for any further processing. model_kwargs: Additional model specific kwargs will be forwarded to the `forward` function of the model. If model is an encoder-decoder model the kwargs should include `encoder_outputs`. @@ -2525,6 +2567,8 @@ def sample( # update generated ids, model inputs, and length for next step input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + if streamer is not None: + streamer.put(next_tokens.cpu()) model_kwargs = self._update_model_kwargs_for_generation( outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder ) @@ -2542,6 +2586,9 @@ def sample( else: this_peer_finished = True + if streamer is not None: + streamer.end() + if return_dict_in_generate: if self.config.is_encoder_decoder: return SampleEncoderDecoderOutput( diff --git a/tests/generation/test_streamers.py b/tests/generation/test_streamers.py new file mode 100644 index 000000000000..120623285904 --- /dev/null +++ b/tests/generation/test_streamers.py @@ -0,0 +1,44 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Team Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a clone of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from transformers import AutoTokenizer, TextStreamer, is_torch_available +from transformers.testing_utils import CaptureStdout, require_torch, torch_device + +from ..test_modeling_common import ids_tensor + + +if is_torch_available(): + from transformers import AutoModelForCausalLM + + +@require_torch +class StreamerTester(unittest.TestCase): + def test_text_streamer_stdout(self): + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") + model = AutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2").to(torch_device) + model.config.eos_token_id = -1 + + input_ids = ids_tensor((1, 5), vocab_size=model.config.vocab_size).to(torch_device) + greedy_ids = model.generate(input_ids, max_new_tokens=10, do_sample=False) + greedy_text = tokenizer.decode(greedy_ids[0]) + + with CaptureStdout() as cs: + streamer = TextStreamer(tokenizer) + model.generate(input_ids, max_new_tokens=10, do_sample=False, streamer=streamer) + + # The greedy text should be printed to stdout, except for the final "\n" in the streamer + self.assertEqual(cs.out[:-1], greedy_text) From 944ff59705d867a36412d416efa2a09e021ccdbf Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Thu, 30 Mar 2023 14:02:16 +0100 Subject: [PATCH 232/277] =?UTF-8?q?=F0=9F=9A=A8=F0=9F=9A=A8=F0=9F=9A=A8=20?= =?UTF-8?q?=20=20Fix=20ordering=20of=20height,=20width=20for=20BLIP=20imag?= =?UTF-8?q?e=20processor=20(#22466)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix ordering of height,width for BLIP --- .../models/blip/image_processing_blip.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 59ea4ac7798a..50808ec65c5d 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -113,24 +113,28 @@ def resize( **kwargs, ) -> np.ndarray: """ - Resize an image. - - Resizes the shorter side of the image to `size["shortest_edge"]` while preserving the aspect ratio. If the - longer side is larger than the max size `(int(`size["shortest_edge"]` * 1333 / 800))`, the longer side is then - resized to the max size while preserving the aspect ratio. + Resize an image to `(size["height"], size["width"])`. Args: image (`np.ndarray`): Image to resize. size (`Dict[str, int]`): - Controls the size of the output image. Should be of the form `{"shortest_edge": int}`. - resample (`PILImageResampling` filter, *optional*, defaults to `PILImageResampling.BICUBIC`): - Resampling filter to use when resiizing the image. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be the same as the input image. + Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): + `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`. + data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The resized image. """ size = get_size_dict(size, default_to_square=True) - output_size = (size["width"], size["height"]) + if "height" not in size or "width" not in size: + raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") + output_size = (size["height"], size["width"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def rescale( From da76ceceb37c170cf998825e414e8b6cdceaec7b Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Thu, 30 Mar 2023 06:16:03 -0700 Subject: [PATCH 233/277] Guard imports of PreTrainedTokenizerFast on is_tokenizers_available (#22285) Guard imports that use the tokenizers library --- src/transformers/models/auto/tokenization_auto.py | 7 ++++++- src/transformers/pipelines/__init__.py | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index fd10c5cd1176..b23b0c39d621 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -24,7 +24,6 @@ from ...dynamic_module_utils import get_class_from_dynamic_module from ...tokenization_utils import PreTrainedTokenizer from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE -from ...tokenization_utils_fast import PreTrainedTokenizerFast from ...utils import cached_file, extract_commit_hash, is_sentencepiece_available, is_tokenizers_available, logging from ..encoder_decoder import EncoderDecoderConfig from .auto_factory import _LazyAutoMapping @@ -37,6 +36,12 @@ ) +if is_tokenizers_available(): + from ...tokenization_utils_fast import PreTrainedTokenizerFast +else: + PreTrainedTokenizerFast = None + + logger = logging.get_logger(__name__) if TYPE_CHECKING: diff --git a/src/transformers/pipelines/__init__.py b/src/transformers/pipelines/__init__.py index c8c0549a4674..7beab782c70d 100755 --- a/src/transformers/pipelines/__init__.py +++ b/src/transformers/pipelines/__init__.py @@ -32,7 +32,6 @@ from ..models.auto.modeling_auto import AutoModelForDepthEstimation from ..models.auto.tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer from ..tokenization_utils import PreTrainedTokenizer -from ..tokenization_utils_fast import PreTrainedTokenizerFast from ..utils import ( HUGGINGFACE_CO_RESOLVE_ENDPOINT, is_kenlm_available, @@ -139,9 +138,13 @@ AutoModelForZeroShotImageClassification, AutoModelForZeroShotObjectDetection, ) + + if TYPE_CHECKING: from ..modeling_tf_utils import TFPreTrainedModel from ..modeling_utils import PreTrainedModel + from ..tokenization_utils_fast import PreTrainedTokenizerFast + logger = logging.get_logger(__name__) @@ -495,7 +498,7 @@ def pipeline( task: str = None, model: Optional = None, config: Optional[Union[str, PretrainedConfig]] = None, - tokenizer: Optional[Union[str, PreTrainedTokenizer, PreTrainedTokenizerFast]] = None, + tokenizer: Optional[Union[str, PreTrainedTokenizer, "PreTrainedTokenizerFast"]] = None, feature_extractor: Optional[Union[str, PreTrainedFeatureExtractor]] = None, image_processor: Optional[Union[str, BaseImageProcessor]] = None, framework: Optional[str] = None, From 6a7b834e7e5703f9c7e6d190d581e13d7d169329 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Thu, 30 Mar 2023 15:36:07 +0200 Subject: [PATCH 234/277] [NLLB-MoE] `model_type` update for auto mapping (#22470) edit default model type and testing path set to hf-internal-testing --- src/transformers/models/nllb_moe/configuration_nllb_moe.py | 2 +- tests/models/nllb_moe/test_modeling_nllb_moe.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/nllb_moe/configuration_nllb_moe.py b/src/transformers/models/nllb_moe/configuration_nllb_moe.py index 03a37bb35d6b..3ff222b93cf6 100644 --- a/src/transformers/models/nllb_moe/configuration_nllb_moe.py +++ b/src/transformers/models/nllb_moe/configuration_nllb_moe.py @@ -125,7 +125,7 @@ class NllbMoeConfig(PretrainedConfig): >>> # Accessing the model configuration >>> configuration = model.config ```""" - model_type = "nllb_moe" + model_type = "nllb-moe" keys_to_ignore_at_inference = ["past_key_values"] attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"} diff --git a/tests/models/nllb_moe/test_modeling_nllb_moe.py b/tests/models/nllb_moe/test_modeling_nllb_moe.py index 567aab56eaea..76cf4c0ea48c 100644 --- a/tests/models/nllb_moe/test_modeling_nllb_moe.py +++ b/tests/models/nllb_moe/test_modeling_nllb_moe.py @@ -354,14 +354,14 @@ def model_inputs(self): @cached_property def tokenizer(self): - return NllbTokenizer.from_pretrained("ArthurZ/random-nllb-moe-2-experts") + return NllbTokenizer.from_pretrained("hf-internal-testing/random-nllb-moe-2-experts") @cached_property def big_model(self): return NllbMoeForConditionalGeneration.from_pretrained("facebook/nllb-moe-54b") def inference_no_head(self): - model = NllbMoeModel.from_pretrained("ArthurZ/random-nllb-moe-2-experts").eval() + model = NllbMoeModel.from_pretrained("hf-internal-testing/random-nllb-moe-2-experts").eval() with torch.no_grad(): output = model(**self.model_inputs) # fmt: off @@ -382,7 +382,7 @@ def test_inference_logits(self): and `transformers` implementation of NLLB-MoE transformers. We only check the logits of the second sample of the batch, as it is padded. """ - model = NllbMoeForConditionalGeneration.from_pretrained("ArthurZ/random-nllb-moe-2-experts").eval() + model = NllbMoeForConditionalGeneration.from_pretrained("hf-internal-testing/random-nllb-moe-2-experts").eval() with torch.no_grad(): output = model(**self.model_inputs) From 1d2f628ed19090f30cb9772e7043545761d69445 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Thu, 30 Mar 2023 15:54:01 +0100 Subject: [PATCH 235/277] Llama: support for `max_position_embeddings` (#22471) * Llama now supports max_position_embeddings * Save config; Cosmetic edits --- .../models/llama/configuration_llama.py | 5 ++ .../models/llama/modeling_llama.py | 49 ++++++------------- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/src/transformers/models/llama/configuration_llama.py b/src/transformers/models/llama/configuration_llama.py index 36b8ab72ab2c..30325b82f787 100644 --- a/src/transformers/models/llama/configuration_llama.py +++ b/src/transformers/models/llama/configuration_llama.py @@ -52,6 +52,9 @@ class LlamaConfig(PretrainedConfig): Number of attention heads for each attention layer in the Transformer encoder. hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. rms_norm_eps (`float`, *optional*, defaults to 1e-12): @@ -85,6 +88,7 @@ def __init__( num_hidden_layers=32, num_attention_heads=32, hidden_act="silu", + max_position_embeddings=2048, initializer_range=0.02, rms_norm_eps=1e-6, use_cache=True, @@ -95,6 +99,7 @@ def __init__( **kwargs, ): self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.num_hidden_layers = num_hidden_layers diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index 6d79536627fc..0c1a189b7a3e 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -160,42 +160,24 @@ def forward(self, x): class LlamaAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" - def __init__( - self, - hidden_size: int, - num_heads: int, - ): + def __init__(self, config: LlamaConfig): super().__init__() - self.hidden_size = hidden_size - self.num_heads = num_heads - self.head_dim = hidden_size // num_heads + self.config = config + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.max_position_embeddings = config.max_position_embeddings - if (self.head_dim * num_heads) != self.hidden_size: + if (self.head_dim * self.num_heads) != self.hidden_size: raise ValueError( f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" - f" and `num_heads`: {num_heads})." + f" and `num_heads`: {self.num_heads})." ) - self.q_proj = nn.Linear( - hidden_size, - num_heads * self.head_dim, - bias=False, - ) - self.k_proj = nn.Linear( - hidden_size, - num_heads * self.head_dim, - bias=False, - ) - self.v_proj = nn.Linear( - hidden_size, - num_heads * self.head_dim, - bias=False, - ) - self.o_proj = nn.Linear( - num_heads * self.head_dim, - hidden_size, - bias=False, - ) - self.rotary_emb = LlamaRotaryEmbedding(self.head_dim) + self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.rotary_emb = LlamaRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -270,10 +252,7 @@ class LlamaDecoderLayer(nn.Module): def __init__(self, config: LlamaConfig): super().__init__() self.hidden_size = config.hidden_size - self.self_attn = LlamaAttention( - hidden_size=self.hidden_size, - num_heads=config.num_attention_heads, - ) + self.self_attn = LlamaAttention(config=config) self.mlp = LlamaMLP( hidden_size=self.hidden_size, intermediate_size=config.intermediate_size, From 49ae6c8b14084d18df1fac17fd59ae367d9821c8 Mon Sep 17 00:00:00 2001 From: Manuel de Prada <6536835+manueldeprada@users.noreply.github.com> Date: Thu, 30 Mar 2023 17:04:12 +0200 Subject: [PATCH 236/277] Docs fix: Multinomial sampling decoding needs "num_beams=1", since by default it is usually not 1. (#22473) Fix: Multinomial sampling needs "num_beams=1", since by default is 5. --- docs/source/en/generation_strategies.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/en/generation_strategies.mdx b/docs/source/en/generation_strategies.mdx index 831c8772b6c6..00ee9221fe68 100644 --- a/docs/source/en/generation_strategies.mdx +++ b/docs/source/en/generation_strategies.mdx @@ -216,11 +216,11 @@ We pride ourselves on being the best in the business and our customer service is ### Multinomial sampling As opposed to greedy search that always chooses a token with the highest probability as the -next token, multinomial sampling randomly selects the next token based on the probability distribution over the entire +next token, multinomial sampling (also called ancestral sampling) randomly selects the next token based on the probability distribution over the entire vocabulary given by the model. Every token with a non-zero probability has a chance of being selected, thus reducing the risk of repetition. -To enable multinomial sampling set `do_sample=True`. +To enable multinomial sampling set `do_sample=True` and `num_beams=1`. ```python >>> from transformers import AutoTokenizer, AutoModelForCausalLM @@ -232,7 +232,7 @@ To enable multinomial sampling set `do_sample=True`. >>> prompt = "Today was an amazing day because" >>> inputs = tokenizer(prompt, return_tensors="pt") ->>> outputs = model.generate(**inputs, do_sample=True, max_new_tokens=100) +>>> outputs = model.generate(**inputs, do_sample=True, num_beams=1, max_new_tokens=100) >>> tokenizer.batch_decode(outputs, skip_special_tokens=True) ['Today was an amazing day because we are now in the final stages of our trip to New York City which was very tough. \ It is a difficult schedule and a challenging part of the year but still worth it. I have been taking things easier and \ From 7492a1e0712c83c19ac78282a807e634e26bf9ee Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 30 Mar 2023 21:06:35 +0200 Subject: [PATCH 237/277] (Re-)Enable Nightly + Past CI (#22393) * Enable Nightly + Past CI * put schedule --------- Co-authored-by: ydshieh --- .github/workflows/build-docker-images.yml | 59 +------- .../build-nightly-ci-docker-images.yml | 75 +++++++++ .../workflows/build-past-ci-docker-images.yml | 61 ++++---- .../workflows/self-nightly-past-ci-caller.yml | 143 ++++++++++++++++++ .github/workflows/self-nightly-scheduled.yml | 34 +++-- .github/workflows/self-past-caller.yml | 136 ----------------- .github/workflows/self-past.yml | 83 +++++++++- .github/workflows/self-scheduled.yml | 3 + docker/transformers-past-gpu/Dockerfile | 23 ++- .../Dockerfile | 42 ++--- utils/get_ci_error_statistics.py | 4 +- utils/notification_service.py | 32 ++-- utils/past_ci_versions.py | 96 +++++------- 13 files changed, 444 insertions(+), 347 deletions(-) create mode 100644 .github/workflows/build-nightly-ci-docker-images.yml create mode 100644 .github/workflows/self-nightly-past-ci-caller.yml delete mode 100644 .github/workflows/self-past-caller.yml diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 6627812a666b..a7df11e8fb84 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -3,7 +3,7 @@ name: Build docker images (scheduled) on: push: branches: - - docker-image* + - build_ci_docker_image* repository_dispatch: workflow_call: inputs: @@ -67,35 +67,6 @@ jobs: push: true tags: huggingface/transformers-all-latest-gpu-push-ci - latest-with-torch-nightly-docker: - name: "Nightly PyTorch + Stable TensorFlow" - # Push CI doesn't need this image - if: inputs.image_postfix != '-push-ci' - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Check out code - uses: actions/checkout@v3 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: ./docker/transformers-all-latest-gpu - build-args: | - REF=main - PYTORCH=pre - push: true - tags: huggingface/transformers-all-latest-torch-nightly-gpu - latest-torch-deepspeed-docker: name: "Latest PyTorch + DeepSpeed" runs-on: ubuntu-latest @@ -153,34 +124,6 @@ jobs: push: true tags: huggingface/transformers-pytorch-deepspeed-latest-gpu-push-ci - nightly-torch-deepspeed-docker: - name: "Nightly PyTorch + DeepSpeed" - # Push CI doesn't need this image - if: inputs.image_postfix != '-push-ci' - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Check out code - uses: actions/checkout@v3 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: ./docker/transformers-pytorch-deepspeed-nightly-gpu - build-args: | - REF=main - push: true - tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu - doc-builder: name: "Doc builder" # Push CI doesn't need this image diff --git a/.github/workflows/build-nightly-ci-docker-images.yml b/.github/workflows/build-nightly-ci-docker-images.yml new file mode 100644 index 000000000000..f13dda7daa82 --- /dev/null +++ b/.github/workflows/build-nightly-ci-docker-images.yml @@ -0,0 +1,75 @@ +name: Build docker images (Nightly CI) + +on: + workflow_call: + push: + branches: + - build_nightly_ci_docker_image* + +concurrency: + group: docker-images-builds + cancel-in-progress: false + +jobs: + latest-with-torch-nightly-docker: + name: "Nightly PyTorch + Stable TensorFlow" + runs-on: ubuntu-latest + steps: + - name: Cleanup disk + run: | + sudo ls -l /usr/local/lib/ + sudo ls -l /usr/share/ + sudo du -sh /usr/local/lib/ + sudo du -sh /usr/share/ + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + sudo du -sh /usr/local/lib/ + sudo du -sh /usr/share/ + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Check out code + uses: actions/checkout@v3 + - + name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v3 + with: + context: ./docker/transformers-all-latest-gpu + build-args: | + REF=main + PYTORCH=pre + push: true + tags: huggingface/transformers-all-latest-torch-nightly-gpu + + nightly-torch-deepspeed-docker: + name: "Nightly PyTorch + DeepSpeed" + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Check out code + uses: actions/checkout@v3 + - + name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v3 + with: + context: ./docker/transformers-pytorch-deepspeed-nightly-gpu + build-args: | + REF=main + push: true + tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu \ No newline at end of file diff --git a/.github/workflows/build-past-ci-docker-images.yml b/.github/workflows/build-past-ci-docker-images.yml index 3a0e1612454c..18d88f2d52fa 100644 --- a/.github/workflows/build-past-ci-docker-images.yml +++ b/.github/workflows/build-past-ci-docker-images.yml @@ -3,7 +3,7 @@ name: Build docker images (Past CI) on: push: branches: - - past-ci-docker-image* + - build_past_ci_docker_image* concurrency: group: docker-images-builds @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - version: ["1.11", "1.10", "1.9", "1.8", "1.7", "1.6", "1.5", "1.4"] + version: ["1.13", "1.12", "1.11", "1.10", "1.9"] runs-on: ubuntu-latest steps: - @@ -24,6 +24,17 @@ jobs: - name: Check out code uses: actions/checkout@v3 + - + id: get-base-image + name: Get Base Image + env: + framework_version: ${{ matrix.version }} + run: | + echo "base_image=$(python3 -c 'import os; from utils.past_ci_versions import past_versions_testing; base_image = past_versions_testing["pytorch"][os.environ["framework_version"]]["base_image"]; print(base_image)')" >> $GITHUB_OUTPUT + - + name: Print Base Image + run: | + echo ${{ steps.get-base-image.outputs.base_image }} - name: Login to DockerHub uses: docker/login-action@v2 @@ -37,6 +48,7 @@ jobs: context: ./docker/transformers-past-gpu build-args: | REF=main + BASE_DOCKER_IMAGE=${{ steps.get-base-image.outputs.base_image }} FRAMEWORK=pytorch VERSION=${{ matrix.version }} push: true @@ -47,7 +59,7 @@ jobs: strategy: fail-fast: false matrix: - version: ["2.8", "2.7", "2.6", "2.5"] + version: ["2.11", "2.10", "2.9", "2.8", "2.7", "2.6", "2.5"] runs-on: ubuntu-latest steps: - @@ -57,37 +69,16 @@ jobs: name: Check out code uses: actions/checkout@v3 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + id: get-base-image + name: Get Base Image + env: + framework_version: ${{ matrix.version }} + run: | + echo "base_image=$(python3 -c 'import os; from utils.past_ci_versions import past_versions_testing; base_image = past_versions_testing["tensorflow"][os.environ["framework_version"]]["base_image"]; print(base_image)')" >> $GITHUB_OUTPUT - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: ./docker/transformers-past-gpu - build-args: | - REF=main - FRAMEWORK=tensorflow - VERSION=${{ matrix.version }} - push: true - tags: huggingface/transformers-tensorflow-past-${{ matrix.version }}-gpu - - past-tensorflow-docker-2-4: - name: "Past TensorFlow Docker" - strategy: - fail-fast: false - matrix: - version: ["2.4"] - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Check out code - uses: actions/checkout@v3 + name: Print Base Image + run: | + echo ${{ steps.get-base-image.outputs.base_image }} - name: Login to DockerHub uses: docker/login-action@v2 @@ -101,8 +92,8 @@ jobs: context: ./docker/transformers-past-gpu build-args: | REF=main - BASE_DOCKER_IMAGE=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 + BASE_DOCKER_IMAGE=${{ steps.get-base-image.outputs.base_image }} FRAMEWORK=tensorflow VERSION=${{ matrix.version }} push: true - tags: huggingface/transformers-tensorflow-past-${{ matrix.version }}-gpu \ No newline at end of file + tags: huggingface/transformers-tensorflow-past-${{ matrix.version }}-gpu diff --git a/.github/workflows/self-nightly-past-ci-caller.yml b/.github/workflows/self-nightly-past-ci-caller.yml new file mode 100644 index 000000000000..e86e6a16662c --- /dev/null +++ b/.github/workflows/self-nightly-past-ci-caller.yml @@ -0,0 +1,143 @@ +name: Self-hosted runner (nightly-past-ci-caller) + +on: + schedule: + # 2 am on each Sunday and Thursday + - cron: "0 2 * * 0,4" + push: + branches: + - run_nightly_ci* + - run_past_ci* + +jobs: + build_nightly_ci_images: + name: Build Nightly CI Docker Images + if: (github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_nightly_ci')) + uses: ./.github/workflows/build-nightly-ci-docker-images.yml + secrets: inherit + + run_nightly_ci: + name: Nightly CI + needs: [build_nightly_ci_images] + uses: ./.github/workflows/self-nightly-scheduled.yml + secrets: inherit + + run_past_ci_pytorch_1-13: + name: PyTorch 1.13 + if: (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))) + needs: [run_nightly_ci] + uses: ./.github/workflows/self-past.yml + with: + framework: pytorch + version: "1.13" + secrets: inherit + + run_past_ci_pytorch_1-12: + name: PyTorch 1.12 + if: (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))) + needs: [run_past_ci_pytorch_1-13] + uses: ./.github/workflows/self-past.yml + with: + framework: pytorch + version: "1.12" + secrets: inherit + + run_past_ci_pytorch_1-11: + name: PyTorch 1.11 + if: (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))) + needs: [run_past_ci_pytorch_1-12] + uses: ./.github/workflows/self-past.yml + with: + framework: pytorch + version: "1.11" + secrets: inherit + + run_past_ci_pytorch_1-10: + name: PyTorch 1.10 + if: (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))) + needs: [run_past_ci_pytorch_1-11] + uses: ./.github/workflows/self-past.yml + with: + framework: pytorch + version: "1.10" + secrets: inherit + + run_past_ci_pytorch_1-9: + name: PyTorch 1.9 + if: (cancelled() != true) && ((github.event_name == 'schedule') || ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci'))) + needs: [run_past_ci_pytorch_1-10] + uses: ./.github/workflows/self-past.yml + with: + framework: pytorch + version: "1.9" + secrets: inherit + + run_past_ci_tensorflow_2-11: + name: TensorFlow 2.11 + if: (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')) + needs: [run_past_ci_pytorch_1-9] + uses: ./.github/workflows/self-past.yml + with: + framework: tensorflow + version: "2.11" + secrets: inherit + + run_past_ci_tensorflow_2-10: + name: TensorFlow 2.10 + if: (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')) + needs: [run_past_ci_tensorflow_2-11] + uses: ./.github/workflows/self-past.yml + with: + framework: tensorflow + version: "2.10" + secrets: inherit + + run_past_ci_tensorflow_2-9: + name: TensorFlow 2.9 + if: (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')) + needs: [run_past_ci_tensorflow_2-10] + uses: ./.github/workflows/self-past.yml + with: + framework: tensorflow + version: "2.9" + secrets: inherit + + run_past_ci_tensorflow_2-8: + name: TensorFlow 2.8 + if: (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')) + needs: [run_past_ci_tensorflow_2-9] + uses: ./.github/workflows/self-past.yml + with: + framework: tensorflow + version: "2.8" + secrets: inherit + + run_past_ci_tensorflow_2-7: + name: TensorFlow 2.7 + if: (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')) + needs: [run_past_ci_tensorflow_2-8] + uses: ./.github/workflows/self-past.yml + with: + framework: tensorflow + version: "2.7" + secrets: inherit + + run_past_ci_tensorflow_2-6: + name: TensorFlow 2.6 + if: (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')) + needs: [run_past_ci_tensorflow_2-7] + uses: ./.github/workflows/self-past.yml + with: + framework: tensorflow + version: "2.6" + secrets: inherit + + run_past_ci_tensorflow_2-5: + name: TensorFlow 2.5 + if: (cancelled() != true) && ((github.event_name == 'push') && startsWith(github.ref_name, 'run_past_ci')) + needs: [run_past_ci_tensorflow_2-6] + uses: ./.github/workflows/self-past.yml + with: + framework: tensorflow + version: "2.5" + secrets: inherit diff --git a/.github/workflows/self-nightly-scheduled.yml b/.github/workflows/self-nightly-scheduled.yml index ca5186e736f4..b3e13cbb1b7d 100644 --- a/.github/workflows/self-nightly-scheduled.yml +++ b/.github/workflows/self-nightly-scheduled.yml @@ -1,4 +1,4 @@ -name: Self-hosted runner (nightly) +name: Self-hosted runner (nightly-ci) # Note that each job's dependencies go into a corresponding docker file. # @@ -8,9 +8,7 @@ name: Self-hosted runner (nightly) on: repository_dispatch: -# Disable temporarily until the test suite can be run under 12 hours. -# schedule: -# - cron: "0 16 * * *" + workflow_call: env: HF_HOME: /mnt/cache @@ -33,7 +31,7 @@ jobs: fetch-depth: 2 - name: Check Runner Status - run: python utils/check_self_hosted_runner.py --target_runners single-gpu-scheduled-ci-runner-docker,multi-gpu-scheduled-ci-runner-docker --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }} + run: python utils/check_self_hosted_runner.py --target_runners single-gpu-past-ci-runner-docker,multi-gpu-past-ci-runner-docker --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }} check_runners: name: Check Runners @@ -41,7 +39,7 @@ jobs: strategy: matrix: machine_type: [single-gpu, multi-gpu] - runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }} + runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker-past-ci') }} container: image: huggingface/transformers-all-latest-torch-nightly-gpu options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ @@ -56,7 +54,7 @@ jobs: strategy: matrix: machine_type: [single-gpu, multi-gpu] - runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }} + runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker-past-ci') }} container: image: huggingface/transformers-all-latest-torch-nightly-gpu options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ @@ -96,7 +94,7 @@ jobs: matrix: folders: ${{ fromJson(needs.setup.outputs.matrix) }} machine_type: [single-gpu] - runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }} + runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker-past-ci') }} container: image: huggingface/transformers-all-latest-torch-nightly-gpu options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ @@ -143,7 +141,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports + name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_nightly path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} run_tests_multi_gpu: @@ -153,7 +151,7 @@ jobs: matrix: folders: ${{ fromJson(needs.setup.outputs.matrix) }} machine_type: [multi-gpu] - runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }} + runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker-past-ci') }} container: image: huggingface/transformers-all-latest-torch-nightly-gpu options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ @@ -200,7 +198,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports + name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_nightly path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} run_all_tests_torch_cuda_extensions_gpu: @@ -209,7 +207,7 @@ jobs: fail-fast: false matrix: machine_type: [single-gpu, multi-gpu] - runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }} + runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker-past-ci') }} needs: setup container: image: huggingface/transformers-pytorch-deepspeed-nightly-gpu @@ -258,7 +256,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports + name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_nightly path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu send_results: @@ -292,7 +290,7 @@ jobs: CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_PAST_FUTURE }} ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} - CI_EVENT: nightly-build + CI_EVENT: Nightly CI RUNNER_STATUS: ${{ needs.check_runner_status.result }} RUNNER_ENV_STATUS: ${{ needs.check_runners.result }} SETUP_STATUS: ${{ needs.setup.result }} @@ -302,3 +300,11 @@ jobs: pip install slack_sdk pip show slack_sdk python utils/notification_service.py "${{ needs.setup.outputs.matrix }}" + + + # delete-artifact + - uses: geekyeggo/delete-artifact@v2 + with: + name: | + single-* + multi-* \ No newline at end of file diff --git a/.github/workflows/self-past-caller.yml b/.github/workflows/self-past-caller.yml deleted file mode 100644 index 2cc81dac8ca2..000000000000 --- a/.github/workflows/self-past-caller.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: Self-hosted runner (past-ci-caller) - -on: - push: - branches: - - run-past-ci* - -jobs: - run_past_ci_pytorch_1-11: - name: PyTorch 1.11 - if: always() - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.11" - secrets: inherit - - run_past_ci_pytorch_1-10: - name: PyTorch 1.10 - if: always() - needs: [run_past_ci_pytorch_1-11] - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.10" - secrets: inherit - - run_past_ci_pytorch_1-9: - name: PyTorch 1.9 - if: always() - needs: [run_past_ci_pytorch_1-10] - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.9" - secrets: inherit - - run_past_ci_pytorch_1-8: - name: PyTorch 1.8 - if: always() - needs: [run_past_ci_pytorch_1-9] - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.8" - secrets: inherit - - run_past_ci_pytorch_1-7: - name: PyTorch 1.7 - if: always() - needs: [run_past_ci_pytorch_1-8] - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.7" - secrets: inherit - - run_past_ci_pytorch_1-6: - name: PyTorch 1.6 - if: always() - needs: [run_past_ci_pytorch_1-7] - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.6" - secrets: inherit - - run_past_ci_pytorch_1-5: - name: PyTorch 1.5 - if: always() - needs: [run_past_ci_pytorch_1-6] - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.5" - secrets: inherit - - run_past_ci_pytorch_1-4: - name: PyTorch 1.4 - if: always() - needs: [run_past_ci_pytorch_1-5] - uses: ./.github/workflows/self-past.yml - with: - framework: pytorch - version: "1.4" - secrets: inherit - - run_past_ci_tensorflow_2-8: - name: TensorFlow 2.8 - if: always() - needs: [run_past_ci_pytorch_1-4] - uses: ./.github/workflows/self-past.yml - with: - framework: tensorflow - version: "2.8" - secrets: inherit - - run_past_ci_tensorflow_2-7: - name: TensorFlow 2.7 - if: always() - needs: [run_past_ci_tensorflow_2-8] - uses: ./.github/workflows/self-past.yml - with: - framework: tensorflow - version: "2.7" - secrets: inherit - - run_past_ci_tensorflow_2-6: - name: TensorFlow 2.6 - if: always() - needs: [run_past_ci_tensorflow_2-7] - uses: ./.github/workflows/self-past.yml - with: - framework: tensorflow - version: "2.6" - secrets: inherit - - run_past_ci_tensorflow_2-5: - name: TensorFlow 2.5 - if: always() - needs: [run_past_ci_tensorflow_2-6] - uses: ./.github/workflows/self-past.yml - with: - framework: tensorflow - version: "2.5" - secrets: inherit - - run_past_ci_tensorflow_2-4: - name: TensorFlow 2.4 - if: always() - needs: [run_past_ci_tensorflow_2-5] - uses: ./.github/workflows/self-past.yml - with: - framework: tensorflow - version: "2.4" - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/self-past.yml b/.github/workflows/self-past.yml index 12ddcc665837..bcb6639a7981 100644 --- a/.github/workflows/self-past.yml +++ b/.github/workflows/self-past.yml @@ -1,4 +1,4 @@ -name: Self-hosted runner (past) +name: Self-hosted runner (past-ci) # Note that each job's dependencies go into a corresponding docker file. # @@ -157,7 +157,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports + name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }} path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} run_tests_multi_gpu: @@ -223,14 +223,80 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports + name: ${{ matrix.machine_type }}_run_all_tests_gpu_${{ env.matrix_folders }}_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }} path: /transformers/reports/${{ matrix.machine_type }}_tests_gpu_${{ matrix.folders }} + run_all_tests_torch_cuda_extensions_gpu: + name: Torch CUDA extension tests + if: inputs.framework == 'pytorch' + strategy: + fail-fast: false + matrix: + machine_type: [single-gpu, multi-gpu] + runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker-past-ci') }} + needs: setup + container: + image: huggingface/transformers-${{ inputs.framework }}-past-${{ inputs.version }}-gpu + options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ + steps: + - name: Update clone + working-directory: /transformers + run: git fetch && git checkout ${{ github.sha }} + + - name: Remove cached torch extensions + run: rm -rf /github/home/.cache/torch_extensions/ + + # To avoid unknown test failures + - name: Pre build DeepSpeed *again* + working-directory: / + run: | + python3 -m pip uninstall -y deepspeed + rm -rf DeepSpeed + git clone https://github.com/microsoft/DeepSpeed && cd DeepSpeed && rm -rf build + DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 python3 -m pip install . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check + + - name: NVIDIA-SMI + run: | + nvidia-smi + + - name: Environment + working-directory: /transformers + run: | + python3 utils/print_env.py + + - name: Show installed libraries and their versions + working-directory: /transformers + run: pip freeze + + - name: Run all tests on GPU + working-directory: /transformers + run: | + python3 -m pytest -v --make-reports=${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu tests/deepspeed tests/extended + + - name: Failure short reports + if: ${{ failure() }} + continue-on-error: true + run: cat /transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu/failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports_postfix_${{ inputs.framework }}-${{ inputs.version }} + path: /transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu + send_results: name: Send results to webhook runs-on: ubuntu-latest if: always() - needs: [check_runner_status, check_runners, setup, run_tests_single_gpu, run_tests_multi_gpu] + needs: [ + check_runner_status, + check_runners, + setup, + run_tests_single_gpu, + run_tests_multi_gpu, + run_all_tests_torch_cuda_extensions_gpu + ] steps: - name: Preliminary job status shell: bash @@ -272,4 +338,11 @@ jobs: uses: actions/upload-artifact@v3 with: name: test_failure_tables_${{ inputs.framework }}-${{ inputs.version }} - path: test_failure_tables \ No newline at end of file + path: test_failure_tables + + # delete-artifact + - uses: geekyeggo/delete-artifact@v2 + with: + name: | + single-* + multi-* \ No newline at end of file diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index f535efba27ca..3ebf38062c42 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -10,6 +10,9 @@ on: repository_dispatch: schedule: - cron: "0 2 * * *" + push: + branches: + - run_scheduled_ci* env: HF_HOME: /mnt/cache diff --git a/docker/transformers-past-gpu/Dockerfile b/docker/transformers-past-gpu/Dockerfile index 99fb550c6a35..8ecc83c339d9 100644 --- a/docker/transformers-past-gpu/Dockerfile +++ b/docker/transformers-past-gpu/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_DOCKER_IMAGE="nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04" +ARG BASE_DOCKER_IMAGE FROM $BASE_DOCKER_IMAGE LABEL maintainer="Hugging Face" @@ -8,7 +8,7 @@ ARG DEBIAN_FRONTEND=noninteractive SHELL ["sh", "-lc"] RUN apt update -RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs +RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs libaio-dev RUN git lfs install RUN python3 -m pip install --no-cache-dir --upgrade pip @@ -23,6 +23,9 @@ RUN cd transformers && python3 setup.py develop ARG FRAMEWORK ARG VERSION +# Control `setuptools` version to avoid some issues +RUN [ "$VERSION" != "1.9" -a "$VERSION" != "1.10" ] && python3 -m pip install -U setuptools || python3 -m pip install -U "setuptools<=59.5" + # Remove all frameworks # (`accelerate` requires `torch`, and this causes import issues for TF-only testing) RUN python3 -m pip uninstall -y torch torchvision torchaudio accelerate tensorflow jax flax @@ -34,4 +37,20 @@ RUN python3 ./transformers/utils/past_ci_versions.py --framework $FRAMEWORK --ve RUN echo "INSTALL_CMD = $INSTALL_CMD" RUN $INSTALL_CMD +RUN [ "$FRAMEWORK" != "pytorch" ] && echo "`deepspeed-testing` installation is skipped" || python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing] + +# Uninstall `torch-tensorrt` and `apex` shipped with the base image +RUN python3 -m pip uninstall -y torch-tensorrt apex + +# Pre-build **nightly** release of DeepSpeed, so it would be ready for testing (otherwise, the 1st deepspeed test will timeout) +RUN python3 -m pip uninstall -y deepspeed +# This has to be run inside the GPU VMs running the tests. (So far, it fails here due to GPU checks during compilation.) +# Issue: https://github.com/microsoft/DeepSpeed/issues/2010 +# RUN git clone https://github.com/microsoft/DeepSpeed && cd DeepSpeed && rm -rf build && \ +# DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 python3 -m pip install . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1 + RUN python3 -m pip install -U "itsdangerous<2.1.0" + +# When installing in editable mode, `transformers` is not recognized as a package. +# this line must be added in order for python to be aware of transformers. +RUN cd transformers && python3 setup.py develop diff --git a/docker/transformers-pytorch-deepspeed-nightly-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-nightly-gpu/Dockerfile index 573e09c22a9c..fcb599ddc232 100644 --- a/docker/transformers-pytorch-deepspeed-nightly-gpu/Dockerfile +++ b/docker/transformers-pytorch-deepspeed-nightly-gpu/Dockerfile @@ -1,10 +1,11 @@ -FROM nvcr.io/nvidia/pytorch:21.03-py3 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel_22-08.html#rel_22-08 +FROM nvcr.io/nvidia/pytorch:22.08-py3 LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive # Example: `cu102`, `cu113`, etc. -ARG CUDA='cu113' +ARG CUDA='cu117' RUN apt -y update RUN apt install -y libaio-dev @@ -20,6 +21,9 @@ RUN python3 -m pip install --no-cache-dir -U --pre torch torchvision torchaudio RUN python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing] +# Uninstall `torch-tensorrt` and `apex` shipped with the base image +RUN python3 -m pip uninstall -y torch-tensorrt apex + # Pre-build **nightly** release of DeepSpeed, so it would be ready for testing (otherwise, the 1st deepspeed test will timeout) RUN python3 -m pip uninstall -y deepspeed # This has to be run inside the GPU VMs running the tests. (So far, it fails here due to GPU checks during compilation.) @@ -27,23 +31,23 @@ RUN python3 -m pip uninstall -y deepspeed # RUN git clone https://github.com/microsoft/DeepSpeed && cd DeepSpeed && rm -rf build && \ # DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 python3 -m pip install . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1 -# For `torchdynamo` tests -# (see https://github.com/huggingface/transformers/pull/17765) -RUN git clone https://github.com/pytorch/functorch -RUN python3 -m pip install --no-cache-dir ./functorch[aot] -RUN cd functorch && python3 setup.py develop - -RUN git clone https://github.com/pytorch/torchdynamo -RUN python3 -m pip install -r ./torchdynamo/requirements.txt -RUN cd torchdynamo && python3 setup.py develop - -# install TensorRT -RUN python3 -m pip install --no-cache-dir -U nvidia-pyindex -RUN python3 -m pip install --no-cache-dir -U nvidia-tensorrt==8.2.4.2 - -# install torch_tensorrt (fx path) -RUN git clone https://github.com/pytorch/TensorRT.git -RUN cd TensorRT/py && python3 setup.py install --fx-only +## For `torchdynamo` tests +## (see https://github.com/huggingface/transformers/pull/17765) +#RUN git clone https://github.com/pytorch/functorch +#RUN python3 -m pip install --no-cache-dir ./functorch[aot] +#RUN cd functorch && python3 setup.py develop +# +#RUN git clone https://github.com/pytorch/torchdynamo +#RUN python3 -m pip install -r ./torchdynamo/requirements.txt +#RUN cd torchdynamo && python3 setup.py develop +# +## install TensorRT +#RUN python3 -m pip install --no-cache-dir -U nvidia-pyindex +#RUN python3 -m pip install --no-cache-dir -U nvidia-tensorrt==8.2.4.2 +# +## install torch_tensorrt (fx path) +#RUN git clone https://github.com/pytorch/TensorRT.git +#RUN cd TensorRT/py && python3 setup.py install --fx-only # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. diff --git a/utils/get_ci_error_statistics.py b/utils/get_ci_error_statistics.py index 09dc4d7dd226..e9dc52b5bbe0 100644 --- a/utils/get_ci_error_statistics.py +++ b/utils/get_ci_error_statistics.py @@ -66,12 +66,12 @@ def get_artifacts_links(worflow_run_id, token=None): def download_artifact(artifact_name, artifact_url, output_dir, token): """Download a GitHub Action artifact from a URL. - The URL is of the from `https://api.github.com/repos/huggingface/transformers/actions/artifacts/{ARTIFACT_ID}/zip`, + The URL is of the form `https://api.github.com/repos/huggingface/transformers/actions/artifacts/{ARTIFACT_ID}/zip`, but it can't be used to download directly. We need to get a redirect URL first. See https://docs.github.com/en/rest/actions/artifacts#download-an-artifact """ # Get the redirect URL first - cmd = f'curl -v -H "Accept: application/vnd.github+json" -H "Authorization: token {token}" {artifact_url}' + cmd = f'curl -v -H "Accept: application/vnd.github+json" -H "Authorization: Bearer {token}" {artifact_url}' output = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) o = output.stdout.decode("utf-8") lines = o.splitlines() diff --git a/utils/notification_service.py b/utils/notification_service.py index 0aefd5844d32..7251b4d400c9 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -590,23 +590,20 @@ def post_reply(self): time.sleep(1) -def retrieve_artifact(name: str, gpu: Optional[str]): +def retrieve_artifact(artifact_path: str, gpu: Optional[str]): if gpu not in [None, "single", "multi"]: raise ValueError(f"Invalid GPU for artifact. Passed GPU: `{gpu}`.") - if gpu is not None: - name = f"{gpu}-gpu_{name}" - _artifact = {} - if os.path.exists(name): - files = os.listdir(name) + if os.path.exists(artifact_path): + files = os.listdir(artifact_path) for file in files: try: - with open(os.path.join(name, file)) as f: + with open(os.path.join(artifact_path, file)) as f: _artifact[file.split(".")[0]] = f.read() except UnicodeDecodeError as e: - raise ValueError(f"Could not open {os.path.join(name, file)}.") from e + raise ValueError(f"Could not open {os.path.join(artifact_path, file)}.") from e return _artifact @@ -629,8 +626,14 @@ def add_path(self, path: str, gpu: str = None): directories = filter(os.path.isdir, os.listdir()) for directory in directories: - if directory.startswith("single-gpu"): - artifact_name = directory[len("single-gpu") + 1 :] + artifact_name = directory + + name_parts = artifact_name.split("_postfix_") + if len(name_parts) > 1: + artifact_name = name_parts[0] + + if artifact_name.startswith("single-gpu"): + artifact_name = artifact_name[len("single-gpu") + 1 :] if artifact_name in _available_artifacts: _available_artifacts[artifact_name].single_gpu = True @@ -639,7 +642,7 @@ def add_path(self, path: str, gpu: str = None): _available_artifacts[artifact_name].add_path(directory, gpu="single") - elif directory.startswith("multi-gpu"): + elif artifact_name.startswith("multi-gpu"): artifact_name = directory[len("multi-gpu") + 1 :] if artifact_name in _available_artifacts: @@ -649,7 +652,6 @@ def add_path(self, path: str, gpu: str = None): _available_artifacts[artifact_name].add_path(directory, gpu="multi") else: - artifact_name = directory if artifact_name not in _available_artifacts: _available_artifacts[artifact_name] = Artifact(artifact_name) @@ -805,10 +807,12 @@ def prepare_reports(title, header, reports, to_truncate=True): framework, version = ci_event.replace("Past CI - ", "").split("-") framework = "PyTorch" if framework == "pytorch" else "TensorFlow" job_name_prefix = f"{framework} {version}" + elif ci_event.startswith("Nightly CI"): + job_name_prefix = "Nightly CI" for model in model_results.keys(): for artifact_path in available_artifacts[f"run_all_tests_gpu_{model}_test_reports"].paths: - artifact = retrieve_artifact(artifact_path["name"], artifact_path["gpu"]) + artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"]) if "stats" in artifact: # Link to the GitHub Action job # The job names use `matrix.folder` which contain things like `models/bert` instead of `models_bert` @@ -901,7 +905,7 @@ def prepare_reports(title, header, reports, to_truncate=True): else: additional_results[key]["job_link"][artifact_path["gpu"]] = github_actions_job_links.get(key) - artifact = retrieve_artifact(artifact_path["name"], artifact_path["gpu"]) + artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"]) stacktraces = handle_stacktraces(artifact["failures_line"]) failed, success, time_spent = handle_test_results(artifact["stats"]) diff --git a/utils/past_ci_versions.py b/utils/past_ci_versions.py index c50bbb9b14c9..61495ab2a46f 100644 --- a/utils/past_ci_versions.py +++ b/utils/past_ci_versions.py @@ -4,6 +4,18 @@ past_versions_testing = { "pytorch": { + "1.13": { + "torch": "1.13.1", + "torchvision": "0.14.1", + "torchaudio": "0.13.1", + "python": 3.9, + "cuda": "cu116", + "install": ( + "python3 -m pip install --no-cache-dir -U torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1" + " --extra-index-url https://download.pytorch.org/whl/cu116" + ), + "base_image": "nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04", + }, "1.12": { "torch": "1.12.1", "torchvision": "0.13.1", @@ -14,6 +26,7 @@ "python3 -m pip install --no-cache-dir -U torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1" " --extra-index-url https://download.pytorch.org/whl/cu113" ), + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, "1.11": { "torch": "1.11.0", @@ -25,6 +38,7 @@ "python3 -m pip install --no-cache-dir -U torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0" " --extra-index-url https://download.pytorch.org/whl/cu113" ), + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, "1.10": { "torch": "1.10.2", @@ -36,6 +50,7 @@ "python3 -m pip install --no-cache-dir -U torch==1.10.2 torchvision==0.11.3 torchaudio==0.10.2" " --extra-index-url https://download.pytorch.org/whl/cu113" ), + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, # torchaudio < 0.10 has no CUDA-enabled binary distributions "1.9": { @@ -48,87 +63,44 @@ "python3 -m pip install --no-cache-dir -U torch==1.9.1 torchvision==0.10.1 torchaudio==0.9.1" " --extra-index-url https://download.pytorch.org/whl/cu111" ), + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, - "1.8": { - "torch": "1.8.1", - "torchvision": "0.9.1", - "torchaudio": "0.8.1", - "python": 3.9, - "cuda": "cu111", - "install": ( - "python3 -m pip install --no-cache-dir -U torch==1.8.1 torchvision==0.9.1 torchaudio==0.8.1" - " --extra-index-url https://download.pytorch.org/whl/cu111" - ), - }, - "1.7": { - "torch": "1.7.1", - "torchvision": "0.8.2", - "torchaudio": "0.7.2", - "python": 3.9, - "cuda": "cu110", - "install": ( - "python3 -m pip install --no-cache-dir -U torch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2" - " --extra-index-url https://download.pytorch.org/whl/cu110" - ), - }, - "1.6": { - "torch": "1.6.0", - "torchvision": "0.7.0", - "torchaudio": "0.6.0", - "python": 3.8, - "cuda": "cu101", - "install": ( - "python3 -m pip install --no-cache-dir -U torch==1.6.0 torchvision==0.7.0 torchaudio==0.6.0" - " --extra-index-url https://download.pytorch.org/whl/cu101" - ), + }, + "tensorflow": { + "2.11": { + "tensorflow": "2.11.1", + "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.11.1", + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, - "1.5": { - "torch": "1.5.1", - "torchvision": "0.6.1", - "torchaudio": "0.5.1", - "python": 3.8, - "cuda": "cu101", - "install": ( - "python3 -m pip install --no-cache-dir -U torch==1.5.1 torchvision==0.6.1 torchaudio==0.5.1" - " --extra-index-url https://download.pytorch.org/whl/cu101" - ), + "2.10": { + "tensorflow": "2.10.1", + "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.10.1", + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, - "1.4": { - "torch": "1.4.0", - "torchvision": "0.5.0", - "torchaudio": "0.4.0", - "python": 3.8, - "cuda": "cu100", - "install": ( - "python3 -m pip install --no-cache-dir -U torch==1.4.0 torchvision==0.5.0 torchaudio==0.4.0" - " --extra-index-url https://download.pytorch.org/whl/cu100" - ), + "2.9": { + "tensorflow": "2.9.3", + "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.9.3", + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, - }, - "tensorflow": { "2.8": { "tensorflow": "2.8.2", "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.8.2", + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, "2.7": { "tensorflow": "2.7.3", "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.7.3", + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, "2.6": { "tensorflow": "2.6.5", "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.6.5", + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, "2.5": { "tensorflow": "2.5.3", "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.5.3", - }, - # need another `nvidia:cuda` docker image, otherwise GPU not working - "2.4": { - "tensorflow": "2.4.4", - "install": "python3 -m pip install --no-cache-dir -U tensorflow==2.4.4", - # This should be specified as a docker build argument. - # We keep the information here for reference only. - "base_docker": "nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04", + "base_image": "nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04", }, }, } From 71422fc48fc569195d29e17d094b49fa6879c0eb Mon Sep 17 00:00:00 2001 From: lewtun Date: Fri, 31 Mar 2023 09:09:40 +0200 Subject: [PATCH 238/277] Relax `eos_token_id < 0` checks in `generate()` from `ValueError` to warning (#22472) * Relax checks from to warning * Fix style * Replace warnings with logger * Use warning vs warn --- src/transformers/generation/logits_process.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/generation/logits_process.py b/src/transformers/generation/logits_process.py index 73e1bdb214e6..95c8064ee404 100644 --- a/src/transformers/generation/logits_process.py +++ b/src/transformers/generation/logits_process.py @@ -106,12 +106,12 @@ class MinLengthLogitsProcessor(LogitsProcessor): def __init__(self, min_length: int, eos_token_id: Union[int, List[int]]): if not isinstance(min_length, int) or min_length < 0: - raise ValueError(f"`min_length` has to be a positive integer, but is {min_length}") + raise ValueError(f"`min_length` has to be a non-negative integer, but is {min_length}") if isinstance(eos_token_id, int): eos_token_id = [eos_token_id] if not all([isinstance(i, int) for i in eos_token_id]) or any([i < 0 for i in eos_token_id]): - raise ValueError(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") + logger.warning(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") self.min_length = min_length self.eos_token_id = eos_token_id @@ -148,7 +148,7 @@ def __init__(self, prompt_length_to_skip: int, min_new_tokens: int, eos_token_id if isinstance(eos_token_id, int): eos_token_id = [eos_token_id] if not all([isinstance(i, int) for i in eos_token_id]) or any([i < 0 for i in eos_token_id]): - raise ValueError(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") + logger.warning(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") self.prompt_length_to_skip = prompt_length_to_skip self.min_new_tokens = min_new_tokens From b37668d8d67d4633f16aa1ce0ab220ec68d495c3 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 31 Mar 2023 14:17:40 +0200 Subject: [PATCH 239/277] Update `Wav2Vec2ProcessorWithLM` doc example (#22474) fix Co-authored-by: ydshieh --- .../models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py b/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py index 16ce6fa035c4..e331da14e810 100644 --- a/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py +++ b/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py @@ -564,7 +564,7 @@ def decode( >>> # compare word offsets with audio `common_voice_en_100038.mp3` online on the dataset viewer: >>> # https://huggingface.co/datasets/common_voice/viewer/en/train >>> word_offsets[:4] - [{'word': 'WHY', 'start_time': 1.42, 'end_time': 1.54}, {'word': 'DOES', 'start_time': 1.64, 'end_time': 1.88}, {'word': 'A', 'start_time': 2.12, 'end_time': 2.14}, {'word': 'MILE', 'start_time': 2.26, 'end_time': 2.46}] + [{'word': 'WHY', 'start_time': 1.42, 'end_time': 1.54}, {'word': 'DOES', 'start_time': 1.66, 'end_time': 1.9}, {'word': 'MILISANDRA', 'start_time': 2.26, 'end_time': 2.9}, {'word': 'LOOK', 'start_time': 3.0, 'end_time': 3.16}] ```""" from pyctcdecode.constants import ( From 3c815cbe6be5a6c72814b848978fc59294ba642a Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 31 Mar 2023 16:07:35 +0200 Subject: [PATCH 240/277] Making sure we can use safetensors to serialize all the time. (#22437) * Making sure we can use safetensors to serialize all the time. * Expanding the tests for increased coverage. * Update the test. * Getting current state of affairs. * Tentative fix. * Fixing black version. * Fixing the worst offenders. * Try to modify less files. * Fixing blip_2 (Weird solution right now). * Fixing deta. * Fix blip ? * Missing extra newline. * No deta modification. * Adding some comments. * Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Addressing comments. * Addressing comments. * creating warn_once. * Warning_once ! --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/modeling_utils.py | 40 +++++++++++++++++++ .../models/blip_2/modeling_blip_2.py | 24 ++++++++++- src/transformers/models/deta/modeling_deta.py | 2 +- .../models/llama/modeling_llama.py | 2 - .../pix2struct/configuration_pix2struct.py | 3 +- tests/test_modeling_common.py | 36 +++++++++++++++++ 6 files changed, 101 insertions(+), 6 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 384876fb6de2..9a6c29c27bdf 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1736,6 +1736,41 @@ def save_pretrained( for ignore_key in self._keys_to_ignore_on_save: if ignore_key in state_dict.keys(): del state_dict[ignore_key] + if safe_serialization: + # Safetensors does not allow tensor aliasing. + # We're going to remove aliases before saving + ptrs = collections.defaultdict(list) + for name, tensor in state_dict.items(): + ptrs[tensor.data_ptr()].append(name) + + # These are all the pointers of shared tensors. + shared_ptrs = {ptr: names for ptr, names in ptrs.items() if len(names) > 1} + warn_names = set() + for names in shared_ptrs.values(): + # Removing the keys which are declared as known duplicates on + # load. This allows to make sure the name which is kept is consistent. + if self._keys_to_ignore_on_load_missing is not None: + for name in names: + matches_pattern = any(re.search(pat, name) for pat in self._keys_to_ignore_on_load_missing) + if matches_pattern and name in state_dict: + del state_dict[name] + + # When not all duplicates have been cleaned, still remove those keys, but put a clear warning. + # If the link between tensors was done at runtime then `from_pretrained` will not get + # the key back leading to random tensor. A proper warning will be shown + # during reload (if applicable), but since the file is not necessarily compatible with + # the config, better show a proper warning. + found = 0 + for name in names: + if name in state_dict: + found += 1 + if found > 1: + del state_dict[name] + warn_names.add(name) + if len(warn_names) > 0: + logger.warning_once( + f"Removed shared tensor {warn_names} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading", + ) # Shard the model if it is too big. weights_name = SAFE_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME @@ -2813,6 +2848,11 @@ def _fix_key(key): missing_keys = list(set(expected_keys) - set(loaded_keys)) unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + # Some tensors maybe have been already filled by another key (tied weights). + existing_ptrs = {model_state_dict[k].data_ptr() for k in loaded_keys if k in model_state_dict} + missing_keys = [ + k for k in missing_keys if k in model_state_dict and model_state_dict[k].data_ptr() not in existing_ptrs + ] # Some models may have keys that are not in the state by design, removing them before needlessly warning # the user. if cls._keys_to_ignore_on_load_missing is not None: diff --git a/src/transformers/models/blip_2/modeling_blip_2.py b/src/transformers/models/blip_2/modeling_blip_2.py index 46f0c9b11ce4..9b00274a0b14 100644 --- a/src/transformers/models/blip_2/modeling_blip_2.py +++ b/src/transformers/models/blip_2/modeling_blip_2.py @@ -1238,8 +1238,28 @@ def __init__(self, config: Blip2Config): # Initialize weights and apply final processing self.post_init() - def get_input_embeddings(self) -> nn.Module: - return self.vision_model.embeddings.patch_embedding + def get_input_embeddings(self): + return self.language_model.get_input_embeddings() + + def set_input_embeddings(self, value): + self.language_model.set_input_embeddings(value) + + def set_output_embeddings(self, new_embeddings): + self.language_model.set_output_embeddings(new_embeddings) + + def get_output_embeddings(self) -> nn.Module: + return self.language_model.get_output_embeddings() + + def get_encoder(self): + return self.language_model.get_encoder() + + def get_decoder(self): + return self.language_model.get_decoder() + + def _tie_weights(self): + if not self.config.use_decoder_only_language_model: + self.language_model.encoder.embed_tokens = self.language_model.shared + self.language_model.decoder.embed_tokens = self.language_model.shared @add_start_docstrings_to_model_forward(BLIP_2_TEXT_INPUTS_DOCSTRING) def get_text_features( diff --git a/src/transformers/models/deta/modeling_deta.py b/src/transformers/models/deta/modeling_deta.py index 6fd2e8fdd184..eabc6e5e690d 100644 --- a/src/transformers/models/deta/modeling_deta.py +++ b/src/transformers/models/deta/modeling_deta.py @@ -244,7 +244,7 @@ class DetaObjectDetectionOutput(ModelOutput): def _get_clones(module, N): - return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + return nn.ModuleList([module for i in range(N)]) def inverse_sigmoid(x, eps=1e-5): diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index 0c1a189b7a3e..c3f5285441bc 100755 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -609,8 +609,6 @@ def custom_forward(*inputs): class LlamaForCausalLM(LlamaPreTrainedModel): - _keys_to_ignore_on_load_missing = [r"lm_head.weight"] - def __init__(self, config): super().__init__(config) self.model = LlamaModel(config) diff --git a/src/transformers/models/pix2struct/configuration_pix2struct.py b/src/transformers/models/pix2struct/configuration_pix2struct.py index 8642602cf97d..dead3d8a0424 100644 --- a/src/transformers/models/pix2struct/configuration_pix2struct.py +++ b/src/transformers/models/pix2struct/configuration_pix2struct.py @@ -357,9 +357,10 @@ def __init__( initializer_factor=1.0, initializer_range=0.02, is_vqa=False, + tie_word_embeddings=False, **kwargs, ): - super().__init__(**kwargs) + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) if text_config is None: text_config = {} diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index f71366d21838..030555aece73 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -27,6 +27,7 @@ import unittest import unittest.mock as mock import warnings +from collections import defaultdict from pathlib import Path from typing import Dict, List, Tuple @@ -1626,6 +1627,41 @@ def check_same_values(layer_1, layer_2): # self.assertTrue(model.transformer.wte.weight.shape, model.lm_head.weight.shape) # self.assertTrue(check_same_values(model.transformer.wte, model.lm_head)) + @require_safetensors + def test_can_use_safetensors(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + for model_class in self.all_model_classes: + model_tied = model_class(config) + with tempfile.TemporaryDirectory() as d: + try: + model_tied.save_pretrained(d, safe_serialization=True) + except Exception as e: + raise Exception(f"Class {model_class.__name__} cannot be saved using safetensors: {e}") + + model_reloaded, infos = model_class.from_pretrained(d, output_loading_info=True) + # Checking the state dicts are correct + reloaded_state = model_reloaded.state_dict() + for k, v in model_tied.state_dict().items(): + self.assertIn(k, reloaded_state, f"Key {k} is missing from reloaded") + torch.testing.assert_close( + v, reloaded_state[k], msg=lambda x: f"{model_class.__name__}: Tensor {k}: {x}" + ) + + # Checking the tensor sharing are correct + ptrs = defaultdict(list) + for k, v in model_tied.state_dict().items(): + ptrs[v.data_ptr()].append(k) + + shared_ptrs = {k: v for k, v in ptrs.items() if len(v) > 1} + + for _, shared_names in shared_ptrs.items(): + reloaded_ptrs = {reloaded_state[k].data_ptr() for k in shared_names} + self.assertEqual( + len(reloaded_ptrs), + 1, + f"The shared pointers are incorrect, found different pointers for keys {shared_names}", + ) + def test_tied_model_weights_key_ignore(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() for model_class in self.all_model_classes: From 76d9d9dc98b8df9073e583d718f91112c57982ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Mar 2023 10:50:33 -0400 Subject: [PATCH 241/277] Bump redis from 4.5.3 to 4.5.4 in /examples/research_projects/decision_transformer (#22494) Bump redis in /examples/research_projects/decision_transformer Bumps [redis](https://github.com/redis/redis-py) from 4.5.3 to 4.5.4. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v4.5.3...v4.5.4) --- updated-dependencies: - dependency-name: redis dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../research_projects/decision_transformer/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/research_projects/decision_transformer/requirements.txt b/examples/research_projects/decision_transformer/requirements.txt index b8545669e2cf..71ac03f908e1 100644 --- a/examples/research_projects/decision_transformer/requirements.txt +++ b/examples/research_projects/decision_transformer/requirements.txt @@ -175,7 +175,7 @@ pytz==2022.1 pytz-deprecation-shim==0.1.0.post0 PyYAML==6.0 ray==1.11.0 -redis==4.5.3 +redis==4.5.4 regex==2022.3.15 requests==2.27.1 requests-oauthlib==1.3.1 From 67308c43717b592d15a4bd1d6371fe9b211cc9d2 Mon Sep 17 00:00:00 2001 From: Sabine Date: Fri, 31 Mar 2023 22:38:34 +0300 Subject: [PATCH 242/277] Update Neptune callback docstring (#22497) * update NeptuneCallback docstring * formatting * apply make style --------- Co-authored-by: Aleksander Wojnarowicz --- src/transformers/integrations.py | 43 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/transformers/integrations.py b/src/transformers/integrations.py index a93eab958197..ce31f9ddd6a2 100644 --- a/src/transformers/integrations.py +++ b/src/transformers/integrations.py @@ -1123,33 +1123,32 @@ def __init__(self): class NeptuneCallback(TrainerCallback): - """TrainerCallback that sends the logs to [Neptune](https://neptune.ai). + """TrainerCallback that sends the logs to [Neptune](https://app.neptune.ai). Args: - api_token (`str`, optional): - Neptune API token obtained upon registration. You can leave this argument out if you have saved your token - to the `NEPTUNE_API_TOKEN` environment variable (strongly recommended). See full setup instructions in the - [docs](https://docs.neptune.ai/getting-started/installation). - project (`str`, optional): - Name of an existing Neptune project, in the form: "workspace-name/project-name". You can find and copy the - name from the project Settings -> Properties in Neptune. If None (default), the value of the - `NEPTUNE_PROJECT` environment variable will be used. - name (`str`, optional): Custom name for the run. + api_token (`str`, *optional*): Neptune API token obtained upon registration. + You can leave this argument out if you have saved your token to the `NEPTUNE_API_TOKEN` environment + variable (strongly recommended). See full setup instructions in the + [docs](https://docs.neptune.ai/setup/installation). + project (`str`, *optional*): Name of an existing Neptune project, in the form "workspace-name/project-name". + You can find and copy the name in Neptune from the project settings -> Properties. If None (default), the + value of the `NEPTUNE_PROJECT` environment variable is used. + name (`str`, *optional*): Custom name for the run. base_namespace (`str`, optional, defaults to "finetuning"): In the Neptune run, the root namespace - that will contain all of the logged metadata. - log_parameters (`bool`, optional, defaults to True): + that will contain all of the metadata logged by the callback. + log_parameters (`bool`, *optional*, defaults to `True`): If True, logs all Trainer arguments and model parameters provided by the Trainer. - log_checkpoints (`str`, optional, defaults to None): - If "same", uploads checkpoints whenever they are saved by the Trainer. If "last", uploads only the most - recently saved checkpoint. If "best", uploads the best checkpoint (among the ones saved by the Trainer). If - None, does not upload checkpoints. - run (`Run`, optional): - Pass a Neptune run object if you want to continue logging to an existing run. Read more about resuming runs - in the [docs](https://docs.neptune.ai/how-to-guides/neptune-api/resume-run). - **neptune_run_kwargs (optional): + log_checkpoints (`str`, *optional*): If "same", uploads checkpoints whenever they are saved by the Trainer. + If "last", uploads only the most recently saved checkpoint. If "best", uploads the best checkpoint (among + the ones saved by the Trainer). If `None`, does not upload checkpoints. + run (`Run`, *optional*): Pass a Neptune run object if you want to continue logging to an existing run. + Read more about resuming runs in the [docs](https://docs.neptune.ai/logging/to_existing_object). + **neptune_run_kwargs (*optional*): Additional keyword arguments to be passed directly to the - [neptune.init_run()](https://docs.neptune.ai/api-reference/neptune#.init_run) function when a new run is - created. + [`neptune.init_run()`](https://docs.neptune.ai/api/neptune#init_run) function when a new run is created. + + For instructions and examples, see the [Transformers integration + guide](https://docs.neptune.ai/integrations/transformers) in the Neptune documentation. """ integration_version_key = "source_code/integrations/transformers" From 6cca3e4c1415e711779a5b8b29bf377bd4ff5989 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Fri, 31 Mar 2023 16:18:43 -0400 Subject: [PATCH 243/277] Test fetch v2 (#22367) * Test fetcher v2 * Fix regexes * Remove sanity check * Fake modification to OPT * Fixes some .sep issues * Remove fake OPT change * Fake modif for BERT * Fake modif for init * Exclude SageMaker tests * Fix test and remove fake modif * Fake setup modif * Fake pipeline modif * Remove all fake modifs * Adds options to skip/force tests * [test-all-models] Fake modif for BERT * Try this way * Does the command actually work? * [test-all-models] Try again! * [skip circleci] Remove fake modif * Remove debug statements * Add the list of important models * Quality * Update utils/tests_fetcher.py Co-authored-by: Lysandre Debut * Address review comments * Address review comments * Fix and add test * Apply suggestions from code review Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> * Address review comments --------- Co-authored-by: Lysandre Debut Co-authored-by: Yih-Dar <2521628+ydshieh@users.noreply.github.com> --- .circleci/config.yml | 1 - Makefile | 1 - tests/repo_utils/test_tests_fetcher.py | 673 ++++++++++++++++++++-- utils/tests_fetcher.py | 750 ++++++++++++------------- 4 files changed, 1009 insertions(+), 416 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7dd66a97d72a..78ed6b02b8e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,7 +176,6 @@ jobs: - run: python utils/check_config_attributes.py - run: python utils/check_doctest_list.py - run: make deps_table_check_updated - - run: python utils/tests_fetcher.py --sanity_check - run: python utils/update_metadata.py --check-only - run: python utils/check_task_guides.py diff --git a/Makefile b/Makefile index 400a35bbfe2e..9e1d197cb647 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,6 @@ repo-consistency: python utils/check_config_docstrings.py python utils/check_config_attributes.py python utils/check_doctest_list.py - python utils/tests_fetcher.py --sanity_check python utils/update_metadata.py --check-only python utils/check_task_guides.py diff --git a/tests/repo_utils/test_tests_fetcher.py b/tests/repo_utils/test_tests_fetcher.py index cd0109b5359d..e02a917700dd 100644 --- a/tests/repo_utils/test_tests_fetcher.py +++ b/tests/repo_utils/test_tests_fetcher.py @@ -13,52 +13,661 @@ # limitations under the License. import os +import shutil import sys +import tempfile import unittest +from contextlib import contextmanager +from pathlib import Path from git import Repo +from transformers.testing_utils import CaptureStdout -git_repo_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) -sys.path.append(os.path.join(git_repo_path, "utils")) -transformers_path = os.path.join(git_repo_path, "src", "transformers") -# Tests are run against this specific commit for reproducibility -# https://github.com/huggingface/transformers/tree/07f6690206e39ed7a4d9dbc58824314f7089bb38 -GIT_TEST_SHA = "07f6690206e39ed7a4d9dbc58824314f7089bb38" +REPO_PATH = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +sys.path.append(os.path.join(REPO_PATH, "utils")) -from tests_fetcher import checkout_commit, clean_code, get_module_dependencies # noqa: E402 +import tests_fetcher # noqa: E402 +from tests_fetcher import ( # noqa: E402 + checkout_commit, + clean_code, + create_module_to_test_map, + create_reverse_dependency_map, + create_reverse_dependency_tree, + diff_is_docstring_only, + extract_imports, + get_all_tests, + get_diff, + get_module_dependencies, + get_tree_starting_at, + infer_tests_to_run, + parse_commit_message, + print_tree_deps_of, +) -class CheckDummiesTester(unittest.TestCase): +BERT_MODELING_FILE = "src/transformers/models/bert/modeling_bert.py" +BERT_MODEL_FILE = """from ...modeling_utils import PreTrainedModel +from ...utils import is_torch_available +from .configuration_bert import BertConfig + +class BertModel: + ''' + This is the docstring. + ''' + This is the code +""" + +BERT_MODEL_FILE_NEW_DOCSTRING = """from ...modeling_utils import PreTrainedModel +from ...utils import is_torch_available +from .configuration_bert import BertConfig + +class BertModel: + ''' + This is the docstring. It has been updated. + ''' + This is the code +""" + +BERT_MODEL_FILE_NEW_CODE = """from ...modeling_utils import PreTrainedModel +from ...utils import is_torch_available +from .configuration_bert import BertConfig + +class BertModel: + ''' + This is the docstring. + ''' + This is the code. It has been updated +""" + + +def create_tmp_repo(tmp_dir, models=None): + """ + Creates a repository in a temporary directory mimicking the structure of Transformers. Uses the list of models + provided (which defaults to just `["bert"]`). + """ + tmp_dir = Path(tmp_dir) + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(exist_ok=True) + repo = Repo.init(tmp_dir) + + if models is None: + models = ["bert"] + class_names = [model[0].upper() + model[1:] for model in models] + + transformers_dir = tmp_dir / "src" / "transformers" + transformers_dir.mkdir(parents=True, exist_ok=True) + with open(transformers_dir / "__init__.py", "w") as f: + init_lines = ["from .utils import cached_file, is_torch_available"] + init_lines.extend( + [f"from .models.{model} import {cls}Config, {cls}Model" for model, cls in zip(models, class_names)] + ) + f.write("\n".join(init_lines) + "\n") + with open(transformers_dir / "configuration_utils.py", "w") as f: + f.write("from .utils import cached_file\n\ncode") + with open(transformers_dir / "modeling_utils.py", "w") as f: + f.write("from .utils import cached_file\n\ncode") + + utils_dir = tmp_dir / "src" / "transformers" / "utils" + utils_dir.mkdir(exist_ok=True) + with open(utils_dir / "__init__.py", "w") as f: + f.write("from .hub import cached_file\nfrom .imports import is_torch_available\n") + with open(utils_dir / "hub.py", "w") as f: + f.write("import huggingface_hub\n\ncode") + with open(utils_dir / "imports.py", "w") as f: + f.write("code") + + model_dir = tmp_dir / "src" / "transformers" / "models" + model_dir.mkdir(parents=True, exist_ok=True) + with open(model_dir / "__init__.py", "w") as f: + f.write("\n".join([f"import {model}" for model in models])) + + for model, cls in zip(models, class_names): + model_dir = tmp_dir / "src" / "transformers" / "models" / model + model_dir.mkdir(parents=True, exist_ok=True) + with open(model_dir / "__init__.py", "w") as f: + f.write(f"from .configuration_{model} import {cls}Config\nfrom .modeling_{model} import {cls}Model\n") + with open(model_dir / f"configuration_{model}.py", "w") as f: + f.write("from ...configuration_utils import PretrainedConfig\ncode") + with open(model_dir / f"modeling_{model}.py", "w") as f: + modeling_code = BERT_MODEL_FILE.replace("bert", model).replace("Bert", cls) + f.write(modeling_code) + + test_dir = tmp_dir / "tests" + test_dir.mkdir(exist_ok=True) + with open(test_dir / "test_modeling_common.py", "w") as f: + f.write("from transformers.modeling_utils import PreTrainedModel\ncode") + + for model, cls in zip(models, class_names): + test_model_dir = test_dir / "models" / model + test_model_dir.mkdir(parents=True, exist_ok=True) + (test_model_dir / "__init__.py").touch() + with open(test_model_dir / f"test_modeling_{model}.py", "w") as f: + f.write( + f"from transformers import {cls}Config, {cls}Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode" + ) + + repo.index.add(["src", "tests"]) + repo.index.commit("Initial commit") + repo.create_head("main") + repo.head.reference = repo.refs.main + repo.delete_head("master") + return repo + + +@contextmanager +def patch_transformer_repo_path(new_folder): + """ + Temporarily patches the variables defines in `tests_fetcher` to use a different location for the repo. + """ + old_repo_path = tests_fetcher.PATH_TO_REPO + tests_fetcher.PATH_TO_REPO = Path(new_folder).resolve() + tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers" + tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests" + try: + yield + finally: + tests_fetcher.PATH_TO_REPO = old_repo_path + tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers" + tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests" + + +def commit_changes(filenames, contents, repo, commit_message="Commit"): + """ + Commit new `contents` to `filenames` inside a given `repo`. + """ + if not isinstance(filenames, list): + filenames = [filenames] + if not isinstance(contents, list): + contents = [contents] + + folder = Path(repo.working_dir) + for filename, content in zip(filenames, contents): + with open(folder / filename, "w") as f: + f.write(content) + repo.index.add(filenames) + commit = repo.index.commit(commit_message) + return commit.hexsha + + +class TestFetcherTester(unittest.TestCase): + def test_checkout_commit(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + repo = create_tmp_repo(tmp_folder) + initial_sha = repo.head.commit.hexsha + new_sha = commit_changes(BERT_MODELING_FILE, BERT_MODEL_FILE_NEW_DOCSTRING, repo) + + assert repo.head.commit.hexsha == new_sha + with checkout_commit(repo, initial_sha): + assert repo.head.commit.hexsha == initial_sha + with open(tmp_folder / BERT_MODELING_FILE) as f: + assert f.read() == BERT_MODEL_FILE + + assert repo.head.commit.hexsha == new_sha + with open(tmp_folder / BERT_MODELING_FILE) as f: + assert f.read() == BERT_MODEL_FILE_NEW_DOCSTRING + def test_clean_code(self): # Clean code removes all strings in triple quotes - self.assertEqual(clean_code('"""\nDocstring\n"""\ncode\n"""Long string"""\ncode\n'), "code\ncode") - self.assertEqual(clean_code("'''\nDocstring\n'''\ncode\n'''Long string'''\ncode\n'''"), "code\ncode") + assert clean_code('"""\nDocstring\n"""\ncode\n"""Long string"""\ncode\n') == "code\ncode" + assert clean_code("'''\nDocstring\n'''\ncode\n'''Long string'''\ncode\n'''") == "code\ncode" # Clean code removes all comments - self.assertEqual(clean_code("code\n# Comment\ncode"), "code\ncode") - self.assertEqual(clean_code("code # inline comment\ncode"), "code \ncode") + assert clean_code("code\n# Comment\ncode") == "code\ncode" + assert clean_code("code # inline comment\ncode") == "code \ncode" - def test_checkout_commit(self): - repo = Repo(git_repo_path) - self.assertNotEqual(repo.head.commit.hexsha, GIT_TEST_SHA) - with checkout_commit(repo, GIT_TEST_SHA): - self.assertEqual(repo.head.commit.hexsha, GIT_TEST_SHA) - self.assertNotEqual(repo.head.commit.hexsha, GIT_TEST_SHA) + def test_get_all_tests(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + with patch_transformer_repo_path(tmp_folder): + assert get_all_tests() == ["tests/models/bert", "tests/test_modeling_common.py"] + + def test_get_all_tests_on_full_repo(self): + all_tests = get_all_tests() + assert "tests/models/albert" in all_tests + assert "tests/models/bert" in all_tests + assert "tests/repo_utils" in all_tests + assert "tests/test_pipeline_mixin.py" in all_tests + assert "tests/models" not in all_tests + assert "tests/__pycache__" not in all_tests + assert "tests/models/albert/test_modeling_albert.py" not in all_tests + assert "tests/repo_utils/test_tests_fetcher.py" not in all_tests + + def test_diff_is_docstring_only(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + repo = create_tmp_repo(tmp_folder) + + branching_point = repo.refs.main.commit + bert_file = BERT_MODELING_FILE + commit_changes(bert_file, BERT_MODEL_FILE_NEW_DOCSTRING, repo) + assert diff_is_docstring_only(repo, branching_point, bert_file) + + commit_changes(bert_file, BERT_MODEL_FILE_NEW_CODE, repo) + assert not diff_is_docstring_only(repo, branching_point, bert_file) + + def test_get_diff(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + repo = create_tmp_repo(tmp_folder) + + initial_commit = repo.refs.main.commit + bert_file = BERT_MODELING_FILE + commit_changes(bert_file, BERT_MODEL_FILE_NEW_DOCSTRING, repo) + assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == [] + + commit_changes(bert_file, BERT_MODEL_FILE_NEW_DOCSTRING + "\n# Adding a comment\n", repo) + assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == [] + + commit_changes(bert_file, BERT_MODEL_FILE_NEW_CODE, repo) + assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == [ + "src/transformers/models/bert/modeling_bert.py" + ] + + commit_changes("src/transformers/utils/hub.py", "import huggingface_hub\n\nnew code", repo) + assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == ["src/transformers/utils/hub.py"] + assert get_diff(repo, repo.head.commit, [initial_commit]) == [ + "src/transformers/models/bert/modeling_bert.py", + "src/transformers/utils/hub.py", + ] + + def test_extract_imports_relative(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + + expected_bert_imports = [ + ("src/transformers/modeling_utils.py", ["PreTrainedModel"]), + ("src/transformers/utils/__init__.py", ["is_torch_available"]), + ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]), + ] + expected_utils_imports = [ + ("src/transformers/utils/hub.py", ["cached_file"]), + ("src/transformers/utils/imports.py", ["is_torch_available"]), + ] + with patch_transformer_repo_path(tmp_folder): + assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports + assert extract_imports("src/transformers/utils/__init__.py") == expected_utils_imports + + with open(tmp_folder / BERT_MODELING_FILE, "w") as f: + f.write( + "from ...utils import cached_file, is_torch_available\nfrom .configuration_bert import BertConfig\n" + ) + expected_bert_imports = [ + ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]), + ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]), + ] + with patch_transformer_repo_path(tmp_folder): + assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports + + # Test with multi-line imports + with open(tmp_folder / BERT_MODELING_FILE, "w") as f: + f.write( + "from ...utils import (\n cached_file,\n is_torch_available\n)\nfrom .configuration_bert import BertConfig\n" + ) + expected_bert_imports = [ + ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]), + ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]), + ] + with patch_transformer_repo_path(tmp_folder): + assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports + + def test_extract_imports_absolute(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + + with open(tmp_folder / BERT_MODELING_FILE, "w") as f: + f.write( + "from transformers.utils import cached_file, is_torch_available\nfrom transformers.models.bert.configuration_bert import BertConfig\n" + ) + expected_bert_imports = [ + ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]), + ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]), + ] + with patch_transformer_repo_path(tmp_folder): + assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports + + # Test with multi-line imports + with open(tmp_folder / BERT_MODELING_FILE, "w") as f: + f.write( + "from transformers.utils import (\n cached_file,\n is_torch_available\n)\nfrom transformers.models.bert.configuration_bert import BertConfig\n" + ) + expected_bert_imports = [ + ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]), + ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]), + ] + with patch_transformer_repo_path(tmp_folder): + assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports + + # Test with base imports + with open(tmp_folder / BERT_MODELING_FILE, "w") as f: + f.write( + "from transformers.utils import (\n cached_file,\n is_torch_available\n)\nfrom transformers import BertConfig\n" + ) + expected_bert_imports = [ + ("src/transformers/__init__.py", ["BertConfig"]), + ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]), + ] + with patch_transformer_repo_path(tmp_folder): + assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports def test_get_module_dependencies(self): - bert_module = os.path.join(transformers_path, "models", "bert", "modeling_bert.py") - expected_deps = [ - "activations.py", - "modeling_outputs.py", - "modeling_utils.py", - "pytorch_utils.py", - "models/bert/configuration_bert.py", - ] - expected_deps = {os.path.join(transformers_path, f) for f in expected_deps} - repo = Repo(git_repo_path) - with checkout_commit(repo, GIT_TEST_SHA): - deps = get_module_dependencies(bert_module) - deps = {os.path.expanduser(f) for f in deps} - self.assertEqual(deps, expected_deps) + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + + expected_bert_dependencies = [ + "src/transformers/modeling_utils.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/utils/imports.py", + ] + with patch_transformer_repo_path(tmp_folder): + assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies + + expected_test_bert_dependencies = [ + "tests/test_modeling_common.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/models/bert/modeling_bert.py", + ] + + with patch_transformer_repo_path(tmp_folder): + assert ( + get_module_dependencies("tests/models/bert/test_modeling_bert.py") + == expected_test_bert_dependencies + ) + + # Test with a submodule + (tmp_folder / "src/transformers/utils/logging.py").touch() + with open(tmp_folder / BERT_MODELING_FILE, "a") as f: + f.write("from ...utils import logging\n") + + expected_bert_dependencies = [ + "src/transformers/modeling_utils.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/utils/logging.py", + "src/transformers/utils/imports.py", + ] + with patch_transformer_repo_path(tmp_folder): + assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies + + # Test with an object non-imported in the init + create_tmp_repo(tmp_folder) + with open(tmp_folder / BERT_MODELING_FILE, "a") as f: + f.write("from ...utils import CONSTANT\n") + + expected_bert_dependencies = [ + "src/transformers/modeling_utils.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/utils/__init__.py", + "src/transformers/utils/imports.py", + ] + with patch_transformer_repo_path(tmp_folder): + assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies + + def test_create_reverse_dependency_tree(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + with patch_transformer_repo_path(tmp_folder): + tree = create_reverse_dependency_tree() + + init_edges = [ + "src/transformers/utils/hub.py", + "src/transformers/utils/imports.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/models/bert/modeling_bert.py", + ] + assert {f for f, g in tree if g == "src/transformers/__init__.py"} == set(init_edges) + + bert_edges = [ + "src/transformers/modeling_utils.py", + "src/transformers/utils/imports.py", + "src/transformers/models/bert/configuration_bert.py", + ] + assert {f for f, g in tree if g == "src/transformers/models/bert/modeling_bert.py"} == set(bert_edges) + + test_bert_edges = [ + "tests/test_modeling_common.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/models/bert/modeling_bert.py", + ] + assert {f for f, g in tree if g == "tests/models/bert/test_modeling_bert.py"} == set(test_bert_edges) + + def test_get_tree_starting_at(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + with patch_transformer_repo_path(tmp_folder): + edges = create_reverse_dependency_tree() + + bert_tree = get_tree_starting_at("src/transformers/models/bert/modeling_bert.py", edges) + config_utils_tree = get_tree_starting_at("src/transformers/configuration_utils.py", edges) + + expected_bert_tree = [ + "src/transformers/models/bert/modeling_bert.py", + [("src/transformers/models/bert/modeling_bert.py", "tests/models/bert/test_modeling_bert.py")], + ] + assert bert_tree == expected_bert_tree + + expected_config_tree = [ + "src/transformers/configuration_utils.py", + [("src/transformers/configuration_utils.py", "src/transformers/models/bert/configuration_bert.py")], + [ + ("src/transformers/models/bert/configuration_bert.py", "tests/models/bert/test_modeling_bert.py"), + ( + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/models/bert/modeling_bert.py", + ), + ], + ] + # Order of the edges is random + assert [set(v) for v in config_utils_tree] == [set(v) for v in expected_config_tree] + + def test_print_tree_deps_of(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + + # There are two possible outputs since the order of the last two lines is non-deterministic. + expected_std_out = """src/transformers/models/bert/modeling_bert.py + tests/models/bert/test_modeling_bert.py +src/transformers/configuration_utils.py + src/transformers/models/bert/configuration_bert.py + src/transformers/models/bert/modeling_bert.py + tests/models/bert/test_modeling_bert.py""" + + expected_std_out_2 = """src/transformers/models/bert/modeling_bert.py + tests/models/bert/test_modeling_bert.py +src/transformers/configuration_utils.py + src/transformers/models/bert/configuration_bert.py + tests/models/bert/test_modeling_bert.py + src/transformers/models/bert/modeling_bert.py""" + + with patch_transformer_repo_path(tmp_folder), CaptureStdout() as cs: + print_tree_deps_of("src/transformers/models/bert/modeling_bert.py") + print_tree_deps_of("src/transformers/configuration_utils.py") + + assert cs.out.strip() in [expected_std_out, expected_std_out_2] + + def test_create_reverse_dependency_map(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + create_tmp_repo(tmp_folder) + with patch_transformer_repo_path(tmp_folder): + reverse_map = create_reverse_dependency_map() + + # impact of BERT modeling file (note that we stop at the inits and don't go down further) + expected_bert_deps = { + "src/transformers/__init__.py", + "src/transformers/models/bert/__init__.py", + "tests/models/bert/test_modeling_bert.py", + } + assert set(reverse_map["src/transformers/models/bert/modeling_bert.py"]) == expected_bert_deps + + # init gets the direct deps (and their recursive deps) + expected_init_deps = { + "src/transformers/utils/__init__.py", + "src/transformers/utils/hub.py", + "src/transformers/utils/imports.py", + "src/transformers/models/bert/__init__.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/models/bert/modeling_bert.py", + "src/transformers/configuration_utils.py", + "src/transformers/modeling_utils.py", + "tests/test_modeling_common.py", + "tests/models/bert/test_modeling_bert.py", + } + assert set(reverse_map["src/transformers/__init__.py"]) == expected_init_deps + + expected_init_deps = { + "src/transformers/__init__.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/models/bert/modeling_bert.py", + "tests/models/bert/test_modeling_bert.py", + } + assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps + + # Test that with more models init of bert only gets deps to bert. + create_tmp_repo(tmp_folder, models=["bert", "gpt2"]) + with patch_transformer_repo_path(tmp_folder): + reverse_map = create_reverse_dependency_map() + + # init gets the direct deps (and their recursive deps) + expected_init_deps = { + "src/transformers/__init__.py", + "src/transformers/models/bert/configuration_bert.py", + "src/transformers/models/bert/modeling_bert.py", + "tests/models/bert/test_modeling_bert.py", + } + assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps + + def test_create_module_to_test_map(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + models = models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)] + create_tmp_repo(tmp_folder, models=models) + with patch_transformer_repo_path(tmp_folder): + test_map = create_module_to_test_map(filter_models=True) + + for model in models: + assert test_map[f"src/transformers/models/{model}/modeling_{model}.py"] == [ + f"tests/models/{model}/test_modeling_{model}.py" + ] + + # Init got filtered + expected_init_tests = { + "tests/test_modeling_common.py", + "tests/models/bert/test_modeling_bert.py", + "tests/models/gpt2/test_modeling_gpt2.py", + } + assert set(test_map["src/transformers/__init__.py"]) == expected_init_tests + + def test_infer_tests_to_run(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)] + repo = create_tmp_repo(tmp_folder, models=models) + + commit_changes("src/transformers/models/bert/modeling_bert.py", BERT_MODEL_FILE_NEW_CODE, repo) + + with patch_transformer_repo_path(tmp_folder): + infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True) + with open(tmp_folder / "test-output.txt", "r") as f: + tests_to_run = f.read() + + assert tests_to_run == "tests/models/bert/test_modeling_bert.py" + + # Fake a new model addition + repo = create_tmp_repo(tmp_folder, models=models) + + branch = repo.create_head("new_model") + branch.checkout() + + with open(tmp_folder / "src/transformers/__init__.py", "a") as f: + f.write("from .models.t5 import T5Config, T5Model\n") + + model_dir = tmp_folder / "src/transformers/models/t5" + model_dir.mkdir(exist_ok=True) + + with open(model_dir / "__init__.py", "w") as f: + f.write("from .configuration_t5 import T5Config\nfrom .modeling_t5 import T5Model\n") + with open(model_dir / "configuration_t5.py", "w") as f: + f.write("from ...configuration_utils import PretrainedConfig\ncode") + with open(model_dir / "modeling_t5.py", "w") as f: + modeling_code = BERT_MODEL_FILE.replace("bert", "t5").replace("Bert", "T5") + f.write(modeling_code) + + test_dir = tmp_folder / "tests/models/t5" + test_dir.mkdir(exist_ok=True) + (test_dir / "__init__.py").touch() + with open(test_dir / "test_modeling_t5.py", "w") as f: + f.write( + "from transformers import T5Config, T5Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode" + ) + + repo.index.add(["src", "tests"]) + repo.index.commit("Add T5 model") + + with patch_transformer_repo_path(tmp_folder): + infer_tests_to_run(tmp_folder / "test-output.txt") + with open(tmp_folder / "test-output.txt", "r") as f: + tests_to_run = f.read() + + expected_tests = { + "tests/models/bert/test_modeling_bert.py", + "tests/models/gpt2/test_modeling_gpt2.py", + "tests/models/t5/test_modeling_t5.py", + "tests/test_modeling_common.py", + } + assert set(tests_to_run.split(" ")) == expected_tests + + with patch_transformer_repo_path(tmp_folder): + infer_tests_to_run(tmp_folder / "test-output.txt", filter_models=False) + with open(tmp_folder / "test-output.txt", "r") as f: + tests_to_run = f.read() + + expected_tests = [f"tests/models/{name}/test_modeling_{name}.py" for name in models + ["t5"]] + expected_tests = set(expected_tests + ["tests/test_modeling_common.py"]) + assert set(tests_to_run.split(" ")) == expected_tests + + def test_infer_tests_to_run_with_test_modifs(self): + with tempfile.TemporaryDirectory() as tmp_folder: + tmp_folder = Path(tmp_folder) + models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)] + repo = create_tmp_repo(tmp_folder, models=models) + + commit_changes( + "tests/models/bert/test_modeling_bert.py", + "from transformers import BertConfig, BertModel\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode1", + repo, + ) + + with patch_transformer_repo_path(tmp_folder): + infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True) + with open(tmp_folder / "test-output.txt", "r") as f: + tests_to_run = f.read() + + assert tests_to_run == "tests/models/bert/test_modeling_bert.py" + + def test_parse_commit_message(self): + assert parse_commit_message("Normal commit") == {"skip": False, "no_filter": False, "test_all": False} + + assert parse_commit_message("[skip ci] commit") == {"skip": True, "no_filter": False, "test_all": False} + assert parse_commit_message("[ci skip] commit") == {"skip": True, "no_filter": False, "test_all": False} + assert parse_commit_message("[skip-ci] commit") == {"skip": True, "no_filter": False, "test_all": False} + assert parse_commit_message("[skip_ci] commit") == {"skip": True, "no_filter": False, "test_all": False} + + assert parse_commit_message("[no filter] commit") == {"skip": False, "no_filter": True, "test_all": False} + assert parse_commit_message("[no-filter] commit") == {"skip": False, "no_filter": True, "test_all": False} + assert parse_commit_message("[no_filter] commit") == {"skip": False, "no_filter": True, "test_all": False} + assert parse_commit_message("[filter-no] commit") == {"skip": False, "no_filter": True, "test_all": False} + + assert parse_commit_message("[test all] commit") == {"skip": False, "no_filter": False, "test_all": True} + assert parse_commit_message("[all test] commit") == {"skip": False, "no_filter": False, "test_all": True} + assert parse_commit_message("[test-all] commit") == {"skip": False, "no_filter": False, "test_all": True} + assert parse_commit_message("[all_test] commit") == {"skip": False, "no_filter": False, "test_all": True} diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 85a8e2e198f8..2a6ac35ce4f9 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -13,6 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Welcome to tests_fetcher V2. +This util is designed to fetch tests to run on a PR so that only the tests impacted by the modifications are run, and +when too many models are being impacted, only run the tests of a subset of core models. It works like this. + +Stage 1: Identify the modified files. This takes all the files from the branching point to the current commit (so +all modifications in a PR, not just the last commit) but excludes modifications that are on docstrings or comments +only. + +Stage 2: Extract the tests to run. This is done by looking at the imports in each module and test file: if module A +imports module B, then changing module B impacts module A, so the tests using module A should be run. We thus get the +dependencies of each model and then recursively builds the 'reverse' map of dependencies to get all modules and tests +impacted by a given file. We then only keep the tests (and only the code models tests if there are too many modules). + +Caveats: + - This module only filters tests by files (not individual tests) so it's better to have tests for different things + in different files. + - This module assumes inits are just importing things, not really building objects, so it's better to structure + them this way and move objects building in separate submodules. +""" + import argparse import collections import json @@ -24,13 +45,36 @@ from git import Repo -# This script is intended to be run from the root of the repo but you can adapt this constant if you need to. -PATH_TO_TRANFORMERS = "." - -# A temporary way to trigger all pipeline tests contained in model test files after PR #21516 -all_model_test_files = [str(x) for x in Path("tests/models/").glob("**/**/test_modeling_*.py")] - -all_pipeline_test_files = [str(x) for x in Path("tests/pipelines/").glob("**/test_pipelines_*.py")] +PATH_TO_REPO = Path(__file__).parent.parent.resolve() +PATH_TO_TRANFORMERS = PATH_TO_REPO / "src/transformers" +PATH_TO_TESTS = PATH_TO_REPO / "tests" + +# List here the models to always test. +IMPORTANT_MODELS = [ + # Most downloaded models + "bert", + "clip", + "t5", + "xlm-roberta", + "gpt2", + "bart", + "mpnet", + "gpt-j", + "wav2vec2", + "deberta-v2", + "layoutlm", + "opt", + "longformer", + "vit", + # Pipeline-specific model (to be sure each pipeline has one model in this list) + "tapas", + "vilt", + "clap", + "detr", + "owlvit", + "dpt", + "videomae", +] @contextmanager @@ -79,17 +123,21 @@ def get_all_tests(): - folders under `tests/models`: `bert`, `gpt2`, etc. - test files under `tests`: `test_modeling_common.py`, `test_tokenization_common.py`, etc. """ - test_root_dir = os.path.join(PATH_TO_TRANFORMERS, "tests") # test folders/files directly under `tests` folder - tests = os.listdir(test_root_dir) - tests = sorted(filter(lambda x: os.path.isdir(x) or x.startswith("tests/test_"), [f"tests/{x}" for x in tests])) + tests = os.listdir(PATH_TO_TESTS) + tests = [f"tests/{f}" for f in tests if "__pycache__" not in f] + tests = sorted([f for f in tests if (PATH_TO_REPO / f).is_dir() or f.startswith("tests/test_")]) # model specific test folders - model_tests_folders = os.listdir(os.path.join(test_root_dir, "models")) - model_test_folders = sorted(filter(os.path.isdir, [f"tests/models/{x}" for x in model_tests_folders])) + model_test_folders = os.listdir(PATH_TO_TESTS / "models") + model_test_folders = [f"tests/models/{f}" for f in model_test_folders if "__pycache__" not in f] + model_test_folders = sorted([f for f in model_test_folders if (PATH_TO_REPO / f).is_dir()]) tests.remove("tests/models") + # Sagemaker tests are not meant to be run on the CI. + if "tests/sagemaker" in tests: + tests.remove("tests/sagemaker") tests = model_test_folders + tests return tests @@ -99,11 +147,12 @@ def diff_is_docstring_only(repo, branching_point, filename): """ Check if the diff is only in docstrings in a filename. """ + folder = Path(repo.working_dir) with checkout_commit(repo, branching_point): - with open(filename, "r", encoding="utf-8") as f: + with open(folder / filename, "r", encoding="utf-8") as f: old_content = f.read() - with open(filename, "r", encoding="utf-8") as f: + with open(folder / filename, "r", encoding="utf-8") as f: new_content = f.read() old_content_clean = clean_code(old_content) @@ -112,31 +161,6 @@ def diff_is_docstring_only(repo, branching_point, filename): return old_content_clean == new_content_clean -def get_modified_python_files(diff_with_last_commit=False): - """ - Return a list of python files that have been modified between: - - - the current head and the main branch if `diff_with_last_commit=False` (default) - - the current head and its parent commit otherwise. - """ - repo = Repo(PATH_TO_TRANFORMERS) - - if not diff_with_last_commit: - print(f"main is at {repo.refs.main.commit}") - print(f"Current head is at {repo.head.commit}") - - branching_commits = repo.merge_base(repo.refs.main, repo.head) - for commit in branching_commits: - print(f"Branching commit: {commit}") - return get_diff(repo, repo.head.commit, branching_commits) - else: - print(f"main is at {repo.head.commit}") - parent_commits = repo.head.commit.parents - for commit in parent_commits: - print(f"Parent commit: {commit}") - return get_diff(repo, repo.head.commit, parent_commits) - - def get_diff(repo, base_commit, commits): """ Get's the diff between one or several commits and the head of the repository. @@ -166,96 +190,173 @@ def get_diff(repo, base_commit, commits): return code_diff -def get_module_dependencies(module_fname): +def get_modified_python_files(diff_with_last_commit=False): + """ + Return a list of python files that have been modified between: + + - the current head and the main branch if `diff_with_last_commit=False` (default) + - the current head and its parent commit otherwise. + """ + repo = Repo(PATH_TO_REPO) + + if not diff_with_last_commit: + print(f"main is at {repo.refs.main.commit}") + print(f"Current head is at {repo.head.commit}") + + branching_commits = repo.merge_base(repo.refs.main, repo.head) + for commit in branching_commits: + print(f"Branching commit: {commit}") + return get_diff(repo, repo.head.commit, branching_commits) + else: + print(f"main is at {repo.head.commit}") + parent_commits = repo.head.commit.parents + for commit in parent_commits: + print(f"Parent commit: {commit}") + return get_diff(repo, repo.head.commit, parent_commits) + + +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+(\.+\S+)\s+import\s+([^\n]+) -> Line only contains from .xxx import yyy and we catch .xxx and yyy +# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every +# other import. +_re_single_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+([^\n]+)(?=\n)") +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\) -> Line continues with from .xxx import (yyy) and we catch .xxx and yyy +# yyy will take multiple lines otherwise there wouldn't be parenthesis. +_re_multi_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\)") +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+transformers(\S*)\s+import\s+([^\n]+) -> Line only contains from transformers.xxx import yyy and we catch +# .xxx and yyy +# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every +# other import. +_re_single_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+transformers(\S*)\s+import\s+([^\n]+)(?=\n)") +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+transformers(\S*)\s+import\s+\(([^\)]+)\) -> Line continues with from transformers.xxx import (yyy) and we +# catch .xxx and yyy. yyy will take multiple lines otherwise there wouldn't be parenthesis. +_re_multi_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+transformers(\S*)\s+import\s+\(([^\)]+)\)") + + +def extract_imports(module_fname, cache=None): """ - Get the dependencies of a module. + Get the imports a given module makes. This takes a module filename and returns the list of module filenames + imported in the module with the objects imported in that module filename. """ - with open(os.path.join(PATH_TO_TRANFORMERS, module_fname), "r", encoding="utf-8") as f: + if cache is not None and module_fname in cache: + return cache[module_fname] + + with open(PATH_TO_REPO / module_fname, "r", encoding="utf-8") as f: content = f.read() - module_parts = module_fname.split(os.path.sep) + # Filter out all docstrings to not get imports in code examples. + splits = content.split('"""') + content = "".join(splits[::2]) + + module_parts = str(module_fname).split(os.path.sep) imported_modules = [] # Let's start with relative imports - relative_imports = re.findall(r"from\s+(\.+\S+)\s+import\s+([^\n]+)\n", content) - relative_imports = [mod for mod, imp in relative_imports if "# tests_ignore" not in imp] - for imp in relative_imports: + relative_imports = _re_single_line_relative_imports.findall(content) + relative_imports = [ + (mod, imp) for mod, imp in relative_imports if "# tests_ignore" not in imp and imp.strip() != "(" + ] + multiline_relative_imports = _re_multi_line_relative_imports.findall(content) + relative_imports += [(mod, imp) for mod, imp in multiline_relative_imports if "# tests_ignore" not in imp] + + for module, imports in relative_imports: level = 0 - while imp.startswith("."): - imp = imp[1:] + while module.startswith("."): + module = module[1:] level += 1 - if len(imp) > 0: - dep_parts = module_parts[: len(module_parts) - level] + imp.split(".") + if len(module) > 0: + dep_parts = module_parts[: len(module_parts) - level] + module.split(".") else: - dep_parts = module_parts[: len(module_parts) - level] + ["__init__.py"] + dep_parts = module_parts[: len(module_parts) - level] imported_module = os.path.sep.join(dep_parts) - # We ignore the main init import as it's only for the __version__ that it's done - # and it would add everything as a dependency. - if not imported_module.endswith("transformers/__init__.py"): - imported_modules.append(imported_module) + imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")])) # Let's continue with direct imports - # The import from the transformers module are ignored for the same reason we ignored the - # main init before. - direct_imports = re.findall(r"from\s+transformers\.(\S+)\s+import\s+([^\n]+)\n", content) - direct_imports = [mod for mod, imp in direct_imports if "# tests_ignore" not in imp] - for imp in direct_imports: - import_parts = imp.split(".") - dep_parts = ["src", "transformers"] + import_parts - imported_modules.append(os.path.sep.join(dep_parts)) + direct_imports = _re_single_line_direct_imports.findall(content) + direct_imports = [(mod, imp) for mod, imp in direct_imports if "# tests_ignore" not in imp and imp.strip() != "("] + multiline_direct_imports = _re_multi_line_direct_imports.findall(content) + direct_imports += [(mod, imp) for mod, imp in multiline_direct_imports if "# tests_ignore" not in imp] - # Now let's just check that we have proper module files, or append an init for submodules - dependencies = [] - for imported_module in imported_modules: - if os.path.isfile(os.path.join(PATH_TO_TRANFORMERS, f"{imported_module}.py")): - dependencies.append(f"{imported_module}.py") - elif os.path.isdir(os.path.join(PATH_TO_TRANFORMERS, imported_module)) and os.path.isfile( - os.path.sep.join([PATH_TO_TRANFORMERS, imported_module, "__init__.py"]) - ): - dependencies.append(os.path.sep.join([imported_module, "__init__.py"])) - return dependencies + for module, imports in direct_imports: + import_parts = module.split(".")[1:] # ignore the first . + dep_parts = ["src", "transformers"] + import_parts + imported_module = os.path.sep.join(dep_parts) + imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")])) + result = [] + for module_file, imports in imported_modules: + if (PATH_TO_REPO / f"{module_file}.py").is_file(): + module_file = f"{module_file}.py" + elif (PATH_TO_REPO / module_file).is_dir() and (PATH_TO_REPO / module_file / "__init__.py").is_file(): + module_file = os.path.sep.join([module_file, "__init__.py"]) + imports = [imp for imp in imports if len(imp) > 0 and re.match("^[A-Za-z0-9_]*$", imp)] + if len(imports) > 0: + result.append((module_file, imports)) -def get_test_dependencies(test_fname): - """ - Get the dependencies of a test file. - """ - with open(os.path.join(PATH_TO_TRANFORMERS, test_fname), "r", encoding="utf-8") as f: - content = f.read() + if cache is not None: + cache[module_fname] = result - # Tests only have relative imports for other test files - # TODO Sylvain: handle relative imports cleanly - relative_imports = re.findall(r"from\s+(\.\S+)\s+import\s+([^\n]+)\n", content) - relative_imports = [test for test, imp in relative_imports if "# tests_ignore" not in imp] + return result - def _convert_relative_import_to_file(relative_import): - level = 0 - while relative_import.startswith("."): - level += 1 - relative_import = relative_import[1:] - directory = os.path.sep.join(test_fname.split(os.path.sep)[:-level]) - return os.path.join(directory, f"{relative_import.replace('.', os.path.sep)}.py") +def get_module_dependencies(module_fname, cache=None): + """ + Get the dependencies of a module from the module filename as a list of module filenames. This will resolve any + __init__ we pass: if we import from a submodule utils, the dependencies will be utils/foo.py and utils/bar.py (if + the objects imported actually come from utils.foo and utils.bar) not utils/__init__.py. + """ + dependencies = [] + imported_modules = extract_imports(module_fname, cache=cache) + # The while loop is to recursively traverse all inits we may encounter. + while len(imported_modules) > 0: + new_modules = [] + for module, imports in imported_modules: + # If we end up in an __init__ we are often not actually importing from this init (except in the case where + # the object is fully defined in the __init__) + if module.endswith("__init__.py"): + # So we get the imports from that init then try to find where our objects come from. + new_imported_modules = extract_imports(module, cache=cache) + for new_module, new_imports in new_imported_modules: + if any([i in new_imports for i in imports]): + if new_module not in dependencies: + new_modules.append((new_module, [i for i in new_imports if i in imports])) + imports = [i for i in imports if i not in new_imports] + if len(imports) > 0: + # If there are any objects lefts, they may be a submodule + path_to_module = PATH_TO_REPO / module.replace("__init__.py", "") + dependencies.extend( + [ + os.path.join(module.replace("__init__.py", ""), f"{i}.py") + for i in imports + if (path_to_module / f"{i}.py").is_file() + ] + ) + imports = [i for i in imports if not (path_to_module / f"{i}.py").is_file()] + if len(imports) > 0: + # Then if there are still objects left, they are fully defined in the init, so we keep it as a + # dependency. + dependencies.append(module) + else: + dependencies.append(module) - dependencies = [_convert_relative_import_to_file(relative_import) for relative_import in relative_imports] - return [f for f in dependencies if os.path.isfile(os.path.join(PATH_TO_TRANFORMERS, f))] + imported_modules = new_modules + return dependencies def create_reverse_dependency_tree(): """ Create a list of all edges (a, b) which mean that modifying a impacts b with a going over all module and test files. """ - modules = [ - str(f.relative_to(PATH_TO_TRANFORMERS)) - for f in (Path(PATH_TO_TRANFORMERS) / "src/transformers").glob("**/*.py") - ] - module_edges = [(d, m) for m in modules for d in get_module_dependencies(m)] - - tests = [str(f.relative_to(PATH_TO_TRANFORMERS)) for f in (Path(PATH_TO_TRANFORMERS) / "tests").glob("**/*.py")] - test_edges = [(d, t) for t in tests for d in get_test_dependencies(t)] + cache = {} + all_modules = list(PATH_TO_TRANFORMERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] + edges = [(dep, mod) for mod in all_modules for dep in get_module_dependencies(mod, cache=cache)] - return module_edges + test_edges + return list(set(edges)) def get_tree_starting_at(module, edges): @@ -264,13 +365,17 @@ def get_tree_starting_at(module, edges): starting at module], [list of edges starting at the preceding level], ...] """ vertices_seen = [module] - new_edges = [edge for edge in edges if edge[0] == module and edge[1] != module] + new_edges = [edge for edge in edges if edge[0] == module and edge[1] != module and "__init__.py" not in edge[1]] tree = [module] while len(new_edges) > 0: tree.append(new_edges) final_vertices = list({edge[1] for edge in new_edges}) vertices_seen.extend(final_vertices) - new_edges = [edge for edge in edges if edge[0] in final_vertices and edge[1] not in vertices_seen] + new_edges = [ + edge + for edge in edges + if edge[0] in final_vertices and edge[1] not in vertices_seen and "__init__.py" not in edge[1] + ] return tree @@ -308,290 +413,159 @@ def create_reverse_dependency_map(): Create the dependency map from module/test filename to the list of modules/tests that depend on it (even recursively). """ - modules = [ - str(f.relative_to(PATH_TO_TRANFORMERS)) - for f in (Path(PATH_TO_TRANFORMERS) / "src/transformers").glob("**/*.py") - ] - # We grab all the dependencies of each module. - direct_deps = {m: get_module_dependencies(m) for m in modules} - - # We add all the dependencies of each test file - tests = [str(f.relative_to(PATH_TO_TRANFORMERS)) for f in (Path(PATH_TO_TRANFORMERS) / "tests").glob("**/*.py")] - direct_deps.update({t: get_test_dependencies(t) for t in tests}) - - all_files = modules + tests + cache = {} + all_modules = list(PATH_TO_TRANFORMERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] + direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules} # This recurses the dependencies something_changed = True while something_changed: something_changed = False - for m in all_files: + for m in all_modules: for d in direct_deps[m]: + if d.endswith("__init__.py"): + continue if d not in direct_deps: raise ValueError(f"KeyError:{d}. From {m}") - for dep in direct_deps[d]: - if dep not in direct_deps[m]: - direct_deps[m].append(dep) - something_changed = True + new_deps = set(direct_deps[d]) - set(direct_deps[m]) + if len(new_deps) > 0: + direct_deps[m].extend(list(new_deps)) + something_changed = True # Finally we can build the reverse map. reverse_map = collections.defaultdict(list) - for m in all_files: - if m.endswith("__init__.py"): - reverse_map[m].extend(direct_deps[m]) + for m in all_modules: for d in direct_deps[m]: reverse_map[d].append(m) + for m in [f for f in all_modules if f.endswith("__init__.py")]: + direct_deps = get_module_dependencies(m, cache=cache) + deps = sum([reverse_map[d] for d in direct_deps if not d.endswith("__init__.py")], direct_deps) + reverse_map[m] = list(set(deps) - {m}) + return reverse_map -# Any module file that has a test name which can't be inferred automatically from its name should go here. A better -# approach is to (re-)name the test file accordingly, and second best to add the correspondence map here. -SPECIAL_MODULE_TO_TEST_MAP = { - "commands/add_new_model_like.py": "utils/test_add_new_model_like.py", - "configuration_utils.py": "test_configuration_common.py", - "convert_graph_to_onnx.py": "onnx/test_onnx.py", - "data/data_collator.py": "trainer/test_data_collator.py", - "deepspeed.py": "deepspeed/", - "feature_extraction_sequence_utils.py": "test_sequence_feature_extraction_common.py", - "feature_extraction_utils.py": "test_feature_extraction_common.py", - "file_utils.py": ["utils/test_file_utils.py", "utils/test_model_output.py"], - "image_processing_utils.py": ["test_image_processing_common.py", "utils/test_image_processing_utils.py"], - "image_transforms.py": "test_image_transforms.py", - "utils/generic.py": ["utils/test_file_utils.py", "utils/test_model_output.py", "utils/test_generic.py"], - "utils/hub.py": "utils/test_hub_utils.py", - "modelcard.py": "utils/test_model_card.py", - "modeling_flax_utils.py": "test_modeling_flax_common.py", - "modeling_tf_utils.py": ["test_modeling_tf_common.py", "utils/test_modeling_tf_core.py"], - "modeling_utils.py": ["test_modeling_common.py", "utils/test_offline.py"], - "models/auto/modeling_auto.py": [ - "models/auto/test_modeling_auto.py", - "models/auto/test_modeling_tf_pytorch.py", - "models/bort/test_modeling_bort.py", - "models/dit/test_modeling_dit.py", - ], - "models/auto/modeling_flax_auto.py": "models/auto/test_modeling_flax_auto.py", - "models/auto/modeling_tf_auto.py": [ - "models/auto/test_modeling_tf_auto.py", - "models/auto/test_modeling_tf_pytorch.py", - "models/bort/test_modeling_tf_bort.py", - ], - "models/gpt2/modeling_gpt2.py": [ - "models/gpt2/test_modeling_gpt2.py", - "models/megatron_gpt2/test_modeling_megatron_gpt2.py", - ], - "models/dpt/modeling_dpt.py": [ - "models/dpt/test_modeling_dpt.py", - "models/dpt/test_modeling_dpt_hybrid.py", - ], - "optimization.py": "optimization/test_optimization.py", - "optimization_tf.py": "optimization/test_optimization_tf.py", - "pipelines/__init__.py": all_pipeline_test_files + all_model_test_files, - "pipelines/base.py": all_pipeline_test_files + all_model_test_files, - "pipelines/text2text_generation.py": [ - "pipelines/test_pipelines_text2text_generation.py", - "pipelines/test_pipelines_summarization.py", - "pipelines/test_pipelines_translation.py", - ], - "pipelines/zero_shot_classification.py": "pipelines/test_pipelines_zero_shot.py", - "testing_utils.py": "utils/test_skip_decorators.py", - "tokenization_utils.py": ["test_tokenization_common.py", "tokenization/test_tokenization_utils.py"], - "tokenization_utils_base.py": ["test_tokenization_common.py", "tokenization/test_tokenization_utils.py"], - "tokenization_utils_fast.py": [ - "test_tokenization_common.py", - "tokenization/test_tokenization_utils.py", - "tokenization/test_tokenization_fast.py", - ], - "trainer.py": [ - "trainer/test_trainer.py", - "extended/test_trainer_ext.py", - "trainer/test_trainer_distributed.py", - "trainer/test_trainer_tpu.py", - ], - "train_pt_utils.py": "trainer/test_trainer_utils.py", - "utils/versions.py": "utils/test_versions_utils.py", -} - - -def module_to_test_file(module_fname): - """ - Returns the name of the file(s) where `module_fname` is tested. - """ - splits = module_fname.split(os.path.sep) - - # Special map has priority - short_name = os.path.sep.join(splits[2:]) - if short_name in SPECIAL_MODULE_TO_TEST_MAP: - test_file = SPECIAL_MODULE_TO_TEST_MAP[short_name] - if isinstance(test_file, str): - return f"tests/{test_file}" - return [f"tests/{f}" for f in test_file] - - module_name = splits[-1] - # Fast tokenizers are tested in the same file as the slow ones. - if module_name.endswith("_fast.py"): - module_name = module_name.replace("_fast.py", ".py") - - # Special case for pipelines submodules - if len(splits) >= 2 and splits[-2] == "pipelines": - default_test_file = f"tests/pipelines/test_pipelines_{module_name}" - return [default_test_file] + all_model_test_files - # Special case for benchmarks submodules - elif len(splits) >= 2 and splits[-2] == "benchmark": - return ["tests/benchmark/test_benchmark.py", "tests/benchmark/test_benchmark_tf.py"] - # Special case for commands submodules - elif len(splits) >= 2 and splits[-2] == "commands": - return "tests/utils/test_cli.py" - # Special case for onnx submodules - elif len(splits) >= 2 and splits[-2] == "onnx": - return ["tests/onnx/test_features.py", "tests/onnx/test_onnx.py", "tests/onnx/test_onnx_v2.py"] - # Special case for utils (not the one in src/transformers, the ones at the root of the repo). - elif len(splits) > 0 and splits[0] == "utils": - default_test_file = f"tests/repo_utils/test_{module_name}" - elif len(splits) > 4 and splits[2] == "models": - default_test_file = f"tests/models/{splits[3]}/test_{module_name}" - elif len(splits) > 2 and splits[2].startswith("generation"): - default_test_file = f"tests/generation/test_{module_name}" - elif len(splits) > 2 and splits[2].startswith("trainer"): - default_test_file = f"tests/trainer/test_{module_name}" - else: - default_test_file = f"tests/utils/test_{module_name}" - - if os.path.isfile(default_test_file): - return default_test_file - - # Processing -> processor - if "processing" in default_test_file: - test_file = default_test_file.replace("processing", "processor") - if os.path.isfile(test_file): - return test_file - - -# This list contains the list of test files we expect never to be launched from a change in a module/util. Those are -# launched separately. -EXPECTED_TEST_FILES_NEVER_TOUCHED = [ - "tests/generation/test_framework_agnostic.py", # Mixins inherited by actual test classes - "tests/mixed_int8/test_mixed_int8.py", # Mixed-int8 bitsandbytes test - "tests/pipelines/test_pipelines_common.py", # Actually checked by the pipeline based file - "tests/sagemaker/test_single_node_gpu.py", # SageMaker test - "tests/sagemaker/test_multi_node_model_parallel.py", # SageMaker test - "tests/sagemaker/test_multi_node_data_parallel.py", # SageMaker test - "tests/test_pipeline_mixin.py", # Contains no test of its own (only the common tester class) - "tests/utils/test_doc_samples.py", # Doc tests -] +def create_module_to_test_map(reverse_map=None, filter_models=False): + """ + Extract the tests from the reverse_dependency_map and potentially filters the model tests. + """ + if reverse_map is None: + reverse_map = create_reverse_dependency_map() + test_map = {module: [f for f in deps if f.startswith("tests")] for module, deps in reverse_map.items()} + if not filter_models: + return test_map -def _print_list(l): - return "\n".join([f"- {f}" for f in l]) + num_model_tests = len(list(PATH_TO_TESTS.glob("models/*"))) + + def has_many_models(tests): + model_tests = {Path(t).parts[2] for t in tests if t.startswith("tests/models/")} + return len(model_tests) > num_model_tests // 2 + + def filter_tests(tests): + return [t for t in tests if not t.startswith("tests/models/") or Path(t).parts[2] in IMPORTANT_MODELS] + + return {module: (filter_tests(tests) if has_many_models(tests) else tests) for module, tests in test_map.items()} -def sanity_check(): +def check_imports_all_exist(): """ - Checks that all test files can be touched by a modification in at least one module/utils. This test ensures that - newly-added test files are properly mapped to some module or utils, so they can be run by the CI. + Isn't used per se by the test fetcher but might be used later as a quality check. Putting this here for now so the + code is not lost. """ - # Grab all module and utils - all_files = [ - str(p.relative_to(PATH_TO_TRANFORMERS)) - for p in (Path(PATH_TO_TRANFORMERS) / "src/transformers").glob("**/*.py") - ] - all_files += [ - str(p.relative_to(PATH_TO_TRANFORMERS)) for p in (Path(PATH_TO_TRANFORMERS) / "utils").glob("**/*.py") - ] + cache = {} + all_modules = list(PATH_TO_TRANFORMERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] + direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules} - # Compute all the test files we get from those. - test_files_found = [] - for f in all_files: - test_f = module_to_test_file(f) - if test_f is not None: - if isinstance(test_f, str): - test_files_found.append(test_f) - else: - test_files_found.extend(test_f) - - # Some of the test files might actually be subfolders so we grab the tests inside. - test_files = [] - for test_f in test_files_found: - if os.path.isdir(os.path.join(PATH_TO_TRANFORMERS, test_f)): - test_files.extend( - [ - str(p.relative_to(PATH_TO_TRANFORMERS)) - for p in (Path(PATH_TO_TRANFORMERS) / test_f).glob("**/test*.py") - ] - ) + for module, deps in direct_deps.items(): + for dep in deps: + if not (PATH_TO_REPO / dep).is_file(): + print(f"{module} has dependency on {dep} which does not exist.") + + +def _print_list(l): + return "\n".join([f"- {f}" for f in l]) + + +def create_json_map(test_files_to_run, json_output_file): + if json_output_file is None: + return + + test_map = {} + for test_file in test_files_to_run: + # `test_file` is a path to a test folder/file, starting with `tests/`. For example, + # - `tests/models/bert/test_modeling_bert.py` or `tests/models/bert` + # - `tests/trainer/test_trainer.py` or `tests/trainer` + # - `tests/test_modeling_common.py` + names = test_file.split(os.path.sep) + if names[1] == "models": + # take the part like `models/bert` for modeling tests + key = os.path.sep.join(names[1:3]) + elif len(names) > 2 or not test_file.endswith(".py"): + # test folders under `tests` or python files under them + # take the part like tokenization, `pipeline`, etc. for other test categories + key = os.path.sep.join(names[1:2]) else: - test_files.append(test_f) + # common test files directly under `tests/` + key = "common" - # Compare to existing test files - existing_test_files = [ - str(p.relative_to(PATH_TO_TRANFORMERS)) for p in (Path(PATH_TO_TRANFORMERS) / "tests").glob("**/test*.py") - ] - not_touched_test_files = [f for f in existing_test_files if f not in test_files] + if key not in test_map: + test_map[key] = [] + test_map[key].append(test_file) - should_be_tested = set(not_touched_test_files) - set(EXPECTED_TEST_FILES_NEVER_TOUCHED) - if len(should_be_tested) > 0: - raise ValueError( - "The following test files are not currently associated with any module or utils files, which means they " - f"will never get run by the CI:\n{_print_list(should_be_tested)}\n. Make sure the names of these test " - "files match the name of the module or utils they are testing, or adapt the constant " - "`SPECIAL_MODULE_TO_TEST_MAP` in `utils/tests_fetcher.py` to add them. If your test file is triggered " - "separately and is not supposed to be run by the regular CI, add it to the " - "`EXPECTED_TEST_FILES_NEVER_TOUCHED` constant instead." - ) + # sort the keys & values + keys = sorted(test_map.keys()) + test_map = {k: " ".join(sorted(test_map[k])) for k in keys} + with open(json_output_file, "w", encoding="UTF-8") as fp: + json.dump(test_map, fp, ensure_ascii=False) -def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None, json_output_file=None): +def infer_tests_to_run( + output_file, diff_with_last_commit=False, filters=None, filter_models=True, json_output_file=None +): modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) print(f"\n### MODIFIED FILES ###\n{_print_list(modified_files)}") # Create the map that will give us all impacted modules. - impacted_modules_map = create_reverse_dependency_map() + reverse_map = create_reverse_dependency_map() impacted_files = modified_files.copy() for f in modified_files: - if f in impacted_modules_map: - impacted_files.extend(impacted_modules_map[f]) + if f in reverse_map: + impacted_files.extend(reverse_map[f]) # Remove duplicates impacted_files = sorted(set(impacted_files)) print(f"\n### IMPACTED FILES ###\n{_print_list(impacted_files)}") # Grab the corresponding test files: - if "setup.py" in impacted_files: + if "setup.py" in modified_files: test_files_to_run = ["tests"] repo_utils_launch = True else: - # Grab the corresponding test files: - test_files_to_run = [] - for f in impacted_files: - # Modified test files are always added - if f.startswith("tests/"): - test_files_to_run.append(f) - # Example files are tested separately - elif f.startswith("examples/pytorch"): - test_files_to_run.append("examples/pytorch/test_pytorch_examples.py") - test_files_to_run.append("examples/pytorch/test_accelerate_examples.py") - elif f.startswith("examples/tensorflow"): - test_files_to_run.append("examples/tensorflow/test_tensorflow_examples.py") - elif f.startswith("examples/flax"): - test_files_to_run.append("examples/flax/test_flax_examples.py") - else: - new_tests = module_to_test_file(f) - if new_tests is not None: - if isinstance(new_tests, str): - test_files_to_run.append(new_tests) - else: - test_files_to_run.extend(new_tests) - - # Remove duplicates + # All modified tests need to be run. + test_files_to_run = [ + f for f in modified_files if f.startswith("tests") and f.split(os.path.sep)[-1].startswith("test") + ] + # Then we grab the corresponding test files. + test_map = create_module_to_test_map(reverse_map=reverse_map, filter_models=filter_models) + for f in modified_files: + if f in test_map: + test_files_to_run.extend(test_map[f]) test_files_to_run = sorted(set(test_files_to_run)) + # Remove SageMaker tests + test_files_to_run = [f for f in test_files_to_run if not f.split(os.path.sep)[1] == "sagemaker"] # Make sure we did not end up with a test file that was removed - test_files_to_run = [f for f in test_files_to_run if os.path.isfile(f) or os.path.isdir(f)] + test_files_to_run = [f for f in test_files_to_run if (PATH_TO_REPO / f).exists()] if filters is not None: filtered_files = [] - for filter in filters: - filtered_files.extend([f for f in test_files_to_run if f.startswith(filter)]) + for _filter in filters: + filtered_files.extend([f for f in test_files_to_run if f.startswith(_filter)]) test_files_to_run = filtered_files - repo_utils_launch = any(f.split(os.path.sep)[1] == "repo_utils" for f in test_files_to_run) + + repo_utils_launch = any(f.split(os.path.sep)[1] == "repo_utils" for f in modified_files) if repo_utils_launch: repo_util_file = Path(output_file).parent / "test_repo_utils.txt" @@ -610,34 +584,7 @@ def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None, j if "tests" in test_files_to_run: test_files_to_run = get_all_tests() - if json_output_file is not None: - test_map = {} - for test_file in test_files_to_run: - # `test_file` is a path to a test folder/file, starting with `tests/`. For example, - # - `tests/models/bert/test_modeling_bert.py` or `tests/models/bert` - # - `tests/trainer/test_trainer.py` or `tests/trainer` - # - `tests/test_modeling_common.py` - names = test_file.split(os.path.sep) - if names[1] == "models": - # take the part like `models/bert` for modeling tests - key = "/".join(names[1:3]) - elif len(names) > 2 or not test_file.endswith(".py"): - # test folders under `tests` or python files under them - # take the part like tokenization, `pipeline`, etc. for other test categories - key = "/".join(names[1:2]) - else: - # common test files directly under `tests/` - key = "common" - - if key not in test_map: - test_map[key] = [] - test_map[key].append(test_file) - - # sort the keys & values - keys = sorted(test_map.keys()) - test_map = {k: " ".join(sorted(test_map[k])) for k in keys} - with open(json_output_file, "w", encoding="UTF-8") as fp: - json.dump(test_map, fp, ensure_ascii=False) + create_json_map(test_files_to_run, json_output_file) def filter_tests(output_file, filters): @@ -667,11 +614,29 @@ def filter_tests(output_file, filters): f.write(" ".join(test_files)) +def parse_commit_message(commit_message): + """ + Parses the commit message to detect if a command is there to skip, force all or part of the CI. + + Returns a dictionary of strings to bools with keys skip, test_all_models and test_all. + """ + if commit_message is None: + return {"skip": False, "no_filter": False, "test_all": False} + + command_search = re.search(r"\[([^\]]*)\]", commit_message) + if command_search is not None: + command = command_search.groups()[0] + command = command.lower().replace("-", " ").replace("_", " ") + skip = command in ["ci skip", "skip ci", "circleci skip", "skip circleci"] + no_filter = set(command.split(" ")) == {"no", "filter"} + test_all = set(command.split(" ")) == {"test", "all"} + return {"skip": skip, "no_filter": no_filter, "test_all": test_all} + else: + return {"skip": False, "no_filter": False, "test_all": False} + + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument( - "--sanity_check", action="store_true", help="Only test that all tests and modules are accounted for." - ) parser.add_argument( "--output_file", type=str, default="test_list.txt", help="Where to store the list of tests to run" ) @@ -704,33 +669,54 @@ def filter_tests(output_file, filters): help="Will only print the tree of modules depending on the file passed.", default=None, ) + parser.add_argument( + "--commit_message", + type=str, + help="The commit message (which could contain a command to force all tests or skip the CI).", + default=None, + ) args = parser.parse_args() if args.print_dependencies_of is not None: print_tree_deps_of(args.print_dependencies_of) - elif args.sanity_check: - sanity_check() elif args.filter_tests: filter_tests(args.output_file, ["pipelines", "repo_utils"]) else: - repo = Repo(PATH_TO_TRANFORMERS) + repo = Repo(PATH_TO_REPO) + commit_message = repo.head.commit.message + commit_flags = parse_commit_message(commit_message) + if commit_flags["skip"]: + print("Force-skipping the CI") + quit() + if commit_flags["no_filter"]: + print("Running all tests fetched without filtering.") + if commit_flags["test_all"]: + print("Force-launching all tests") diff_with_last_commit = args.diff_with_last_commit if not diff_with_last_commit and not repo.head.is_detached and repo.head.ref == repo.refs.main: print("main branch detected, fetching tests against last commit.") diff_with_last_commit = True - try: - infer_tests_to_run( - args.output_file, - diff_with_last_commit=diff_with_last_commit, - filters=args.filters, - json_output_file=args.json_output_file, - ) - filter_tests(args.output_file, ["repo_utils"]) - except Exception as e: - print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.") + if not commit_flags["test_all"]: + try: + infer_tests_to_run( + args.output_file, + diff_with_last_commit=diff_with_last_commit, + filters=args.filters, + json_output_file=args.json_output_file, + filter_models=not commit_flags["no_filter"], + ) + filter_tests(args.output_file, ["repo_utils"]) + except Exception as e: + print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.") + commit_flags["test_all"] = True + + if commit_flags["test_all"]: with open(args.output_file, "w", encoding="utf-8") as f: if args.filters is None: f.write("./tests/") else: f.write(" ".join(args.filters)) + + test_files_to_run = get_all_tests() + create_json_map(test_files_to_run, args.json_output_file) From 29e8140236ecc9368002f017142b03046f39041b Mon Sep 17 00:00:00 2001 From: kevinpro <43642508+Ricardokevins@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:41:39 +0800 Subject: [PATCH 244/277] Update convert_llama_weights_to_hf.py (#22525) --- src/transformers/models/llama/convert_llama_weights_to_hf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/llama/convert_llama_weights_to_hf.py b/src/transformers/models/llama/convert_llama_weights_to_hf.py index b5612777936b..3ba48f7c6fe4 100644 --- a/src/transformers/models/llama/convert_llama_weights_to_hf.py +++ b/src/transformers/models/llama/convert_llama_weights_to_hf.py @@ -34,7 +34,7 @@ Thereafter, models can be loaded via: ```py -from transformers import LlamaForCausalLM, LlamaForTokenizer +from transformers import LlamaForCausalLM, LlamaTokenizer model = LlamaForCausalLM.from_pretrained("/output/path") tokenizer = LlamaTokenizer.from_pretrained("/output/path") From 89c25574b597f3856e3ed5297bfacc9b3cb5a9cb Mon Sep 17 00:00:00 2001 From: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:06:25 +0100 Subject: [PATCH 245/277] Backbone add out indices (#22493) * Add out_indices to backbones, deprecate out_features * Update - can specify both out_features and out_indices but not both * Can specify both * Fix copies * Add out_indices to convnextv2 configuration --- .../models/bit/configuration_bit.py | 31 ++++++++++++++++++- src/transformers/models/bit/modeling_bit.py | 4 +++ .../models/convnext/configuration_convnext.py | 31 ++++++++++++++++++- .../models/convnext/modeling_convnext.py | 4 +++ .../convnextv2/configuration_convnextv2.py | 31 ++++++++++++++++++- .../models/convnextv2/modeling_convnextv2.py | 4 +++ .../models/dinat/configuration_dinat.py | 31 ++++++++++++++++++- .../models/dinat/modeling_dinat.py | 4 +++ .../configuration_maskformer_swin.py | 31 ++++++++++++++++++- .../maskformer/modeling_maskformer_swin.py | 4 +++ .../models/nat/configuration_nat.py | 31 ++++++++++++++++++- src/transformers/models/nat/modeling_nat.py | 4 +++ .../models/resnet/configuration_resnet.py | 31 ++++++++++++++++++- .../models/resnet/modeling_resnet.py | 4 +++ .../models/swin/configuration_swin.py | 31 ++++++++++++++++++- src/transformers/models/swin/modeling_swin.py | 4 +++ 16 files changed, 272 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/bit/configuration_bit.py b/src/transformers/models/bit/configuration_bit.py index 278c7f1c7f1a..da53807f3f06 100644 --- a/src/transformers/models/bit/configuration_bit.py +++ b/src/transformers/models/bit/configuration_bit.py @@ -63,7 +63,12 @@ class BitConfig(PretrainedConfig): The width factor for the model. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: ```python @@ -98,6 +103,7 @@ def __init__( output_stride=32, width_factor=1, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -122,6 +128,21 @@ def __init__( self.width_factor = width_factor self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -130,4 +151,12 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices diff --git a/src/transformers/models/bit/modeling_bit.py b/src/transformers/models/bit/modeling_bit.py index b149bf412a76..6a63a7a3e29c 100644 --- a/src/transformers/models/bit/modeling_bit.py +++ b/src/transformers/models/bit/modeling_bit.py @@ -850,6 +850,10 @@ def __init__(self, config): self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] self.num_features = [config.embedding_size] + config.hidden_sizes + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) # initialize weights and apply final processing self.post_init() diff --git a/src/transformers/models/convnext/configuration_convnext.py b/src/transformers/models/convnext/configuration_convnext.py index d4807bc5741a..a4b7272295c7 100644 --- a/src/transformers/models/convnext/configuration_convnext.py +++ b/src/transformers/models/convnext/configuration_convnext.py @@ -66,7 +66,12 @@ class ConvNextConfig(PretrainedConfig): The drop rate for stochastic depth. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: ```python @@ -97,6 +102,7 @@ def __init__( drop_path_rate=0.0, image_size=224, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -113,6 +119,21 @@ def __init__( self.drop_path_rate = drop_path_rate self.image_size = image_size self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(self.depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -121,7 +142,15 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices class ConvNextOnnxConfig(OnnxConfig): diff --git a/src/transformers/models/convnext/modeling_convnext.py b/src/transformers/models/convnext/modeling_convnext.py index 7076e64ccbe9..35302bffed66 100755 --- a/src/transformers/models/convnext/modeling_convnext.py +++ b/src/transformers/models/convnext/modeling_convnext.py @@ -487,6 +487,10 @@ def __init__(self, config): self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] self.num_features = [config.hidden_sizes[0]] + config.hidden_sizes + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) # Add layer norms to hidden states of out_features hidden_states_norms = {} diff --git a/src/transformers/models/convnextv2/configuration_convnextv2.py b/src/transformers/models/convnextv2/configuration_convnextv2.py index 4c79d59b104b..f02a21371b20 100644 --- a/src/transformers/models/convnextv2/configuration_convnextv2.py +++ b/src/transformers/models/convnextv2/configuration_convnextv2.py @@ -58,7 +58,12 @@ class ConvNextV2Config(PretrainedConfig): The drop rate for stochastic depth. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: ```python @@ -88,6 +93,7 @@ def __init__( drop_path_rate=0.0, image_size=224, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -103,6 +109,21 @@ def __init__( self.drop_path_rate = drop_path_rate self.image_size = image_size self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(self.depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -111,4 +132,12 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices diff --git a/src/transformers/models/convnextv2/modeling_convnextv2.py b/src/transformers/models/convnextv2/modeling_convnextv2.py index 741049af8e98..5fbf7831ef12 100644 --- a/src/transformers/models/convnextv2/modeling_convnextv2.py +++ b/src/transformers/models/convnextv2/modeling_convnextv2.py @@ -510,6 +510,10 @@ def __init__(self, config): self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] self.num_features = [config.hidden_sizes[0]] + config.hidden_sizes + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) # Add layer norms to hidden states of out_features hidden_states_norms = {} diff --git a/src/transformers/models/dinat/configuration_dinat.py b/src/transformers/models/dinat/configuration_dinat.py index 1d60628f3de6..7c6a84ecdd13 100644 --- a/src/transformers/models/dinat/configuration_dinat.py +++ b/src/transformers/models/dinat/configuration_dinat.py @@ -72,7 +72,12 @@ class DinatConfig(PretrainedConfig): The initial value for the layer scale. Disabled if <=0. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: @@ -114,6 +119,7 @@ def __init__( layer_norm_eps=1e-5, layer_scale_init_value=0.0, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -139,6 +145,21 @@ def __init__( self.hidden_size = int(embed_dim * 2 ** (len(depths) - 1)) self.layer_scale_init_value = layer_scale_init_value self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -147,4 +168,12 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices diff --git a/src/transformers/models/dinat/modeling_dinat.py b/src/transformers/models/dinat/modeling_dinat.py index 3e102d1052dd..5b2394f122f1 100644 --- a/src/transformers/models/dinat/modeling_dinat.py +++ b/src/transformers/models/dinat/modeling_dinat.py @@ -891,6 +891,10 @@ def __init__(self, config): self.encoder = DinatEncoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] # Add layer norms to hidden states of out_features diff --git a/src/transformers/models/maskformer/configuration_maskformer_swin.py b/src/transformers/models/maskformer/configuration_maskformer_swin.py index d653b3c781b1..ca60b6176eed 100644 --- a/src/transformers/models/maskformer/configuration_maskformer_swin.py +++ b/src/transformers/models/maskformer/configuration_maskformer_swin.py @@ -68,7 +68,12 @@ class MaskFormerSwinConfig(PretrainedConfig): The epsilon used by the layer normalization layers. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: @@ -110,6 +115,7 @@ def __init__( initializer_range=0.02, layer_norm_eps=1e-5, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -135,6 +141,21 @@ def __init__( # this indicates the channel dimension after the last stage of the model self.hidden_size = int(embed_dim * 2 ** (len(depths) - 1)) self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -143,4 +164,12 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices diff --git a/src/transformers/models/maskformer/modeling_maskformer_swin.py b/src/transformers/models/maskformer/modeling_maskformer_swin.py index e84f376c045e..743856ca0b99 100644 --- a/src/transformers/models/maskformer/modeling_maskformer_swin.py +++ b/src/transformers/models/maskformer/modeling_maskformer_swin.py @@ -859,6 +859,10 @@ def __init__(self, config: MaskFormerSwinConfig): if "stem" in self.out_features: raise ValueError("This backbone does not support 'stem' in the `out_features`.") + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] self.hidden_states_norms = nn.ModuleList([nn.LayerNorm(num_channels) for num_channels in self.channels]) diff --git a/src/transformers/models/nat/configuration_nat.py b/src/transformers/models/nat/configuration_nat.py index 83c9e8f82337..a74b8c9165c7 100644 --- a/src/transformers/models/nat/configuration_nat.py +++ b/src/transformers/models/nat/configuration_nat.py @@ -70,7 +70,12 @@ class NatConfig(PretrainedConfig): The initial value for the layer scale. Disabled if <=0. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: @@ -111,6 +116,7 @@ def __init__( layer_norm_eps=1e-5, layer_scale_init_value=0.0, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -135,6 +141,21 @@ def __init__( self.hidden_size = int(embed_dim * 2 ** (len(depths) - 1)) self.layer_scale_init_value = layer_scale_init_value self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -143,4 +164,12 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices diff --git a/src/transformers/models/nat/modeling_nat.py b/src/transformers/models/nat/modeling_nat.py index fcc05947ed1b..c5e83b29da67 100644 --- a/src/transformers/models/nat/modeling_nat.py +++ b/src/transformers/models/nat/modeling_nat.py @@ -869,6 +869,10 @@ def __init__(self, config): self.encoder = NatEncoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] # Add layer norms to hidden states of out_features diff --git a/src/transformers/models/resnet/configuration_resnet.py b/src/transformers/models/resnet/configuration_resnet.py index bf1855340b07..6a88935f3b02 100644 --- a/src/transformers/models/resnet/configuration_resnet.py +++ b/src/transformers/models/resnet/configuration_resnet.py @@ -60,7 +60,12 @@ class ResNetConfig(PretrainedConfig): If `True`, the first stage will downsample the inputs using a `stride` of 2. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: ```python @@ -89,6 +94,7 @@ def __init__( hidden_act="relu", downsample_in_first_stage=False, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -102,6 +108,21 @@ def __init__( self.hidden_act = hidden_act self.downsample_in_first_stage = downsample_in_first_stage self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -110,7 +131,15 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices class ResNetOnnxConfig(OnnxConfig): diff --git a/src/transformers/models/resnet/modeling_resnet.py b/src/transformers/models/resnet/modeling_resnet.py index 946d6b9a0ece..6926f6a43116 100644 --- a/src/transformers/models/resnet/modeling_resnet.py +++ b/src/transformers/models/resnet/modeling_resnet.py @@ -437,6 +437,10 @@ def __init__(self, config): self.encoder = ResNetEncoder(config) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) self.num_features = [config.embedding_size] + config.hidden_sizes # initialize weights and apply final processing diff --git a/src/transformers/models/swin/configuration_swin.py b/src/transformers/models/swin/configuration_swin.py index 4f4625ad0e7f..612bc9949fb2 100644 --- a/src/transformers/models/swin/configuration_swin.py +++ b/src/transformers/models/swin/configuration_swin.py @@ -83,7 +83,12 @@ class SwinConfig(PretrainedConfig): Factor to increase the spatial resolution by in the decoder head for masked image modeling. out_features (`List[str]`, *optional*): If used as backbone, list of features to output. Can be any of `"stem"`, `"stage1"`, `"stage2"`, etc. - (depending on how many stages the model has). Will default to the last stage if unset. + (depending on how many stages the model has). If unset and `out_indices` is set, will default to the + corresponding stages. If unset and `out_indices` is unset, will default to the last stage. + out_indices (`List[int]`, *optional*): + If used as backbone, list of indices of features to output. Can be any of 0, 1, 2, etc. (depending on how + many stages the model has). If unset and `out_features` is set, will default to the corresponding stages. + If unset and `out_features` is unset, will default to the last stage. Example: @@ -126,6 +131,7 @@ def __init__( layer_norm_eps=1e-5, encoder_stride=32, out_features=None, + out_indices=None, **kwargs, ): super().__init__(**kwargs) @@ -152,6 +158,21 @@ def __init__( # this indicates the channel dimension after the last stage of the model self.hidden_size = int(embed_dim * 2 ** (len(depths) - 1)) self.stage_names = ["stem"] + [f"stage{idx}" for idx in range(1, len(depths) + 1)] + + if out_features is not None and out_indices is not None: + if len(out_features) != len(out_indices): + raise ValueError("out_features and out_indices should have the same length if both are set") + elif out_features != [self.stage_names[idx] for idx in out_indices]: + raise ValueError("out_features and out_indices should correspond to the same stages if both are set") + + if out_features is None and out_indices is not None: + out_features = [self.stage_names[idx] for idx in out_indices] + elif out_features is not None and out_indices is None: + out_indices = [self.stage_names.index(feature) for feature in out_features] + elif out_features is None and out_indices is None: + out_features = [self.stage_names[-1]] + out_indices = [len(self.stage_names) - 1] + if out_features is not None: if not isinstance(out_features, list): raise ValueError("out_features should be a list") @@ -160,7 +181,15 @@ def __init__( raise ValueError( f"Feature {feature} is not a valid feature name. Valid names are {self.stage_names}" ) + if out_indices is not None: + if not isinstance(out_indices, (list, tuple)): + raise ValueError("out_indices should be a list or tuple") + for idx in out_indices: + if idx >= len(self.stage_names): + raise ValueError(f"Index {idx} is not a valid index for a list of length {len(self.stage_names)}") + self.out_features = out_features + self.out_indices = out_indices class SwinOnnxConfig(OnnxConfig): diff --git a/src/transformers/models/swin/modeling_swin.py b/src/transformers/models/swin/modeling_swin.py index 8c1c7a0b655f..520c215d4634 100644 --- a/src/transformers/models/swin/modeling_swin.py +++ b/src/transformers/models/swin/modeling_swin.py @@ -1255,6 +1255,10 @@ def __init__(self, config: SwinConfig): self.encoder = SwinEncoder(config, self.embeddings.patch_grid) self.out_features = config.out_features if config.out_features is not None else [self.stage_names[-1]] + if config.out_indices is not None: + self.out_indices = config.out_indices + else: + self.out_indices = tuple(i for i, layer in enumerate(self.stage_names) if layer in self.out_features) self.num_features = [config.embed_dim] + [int(config.embed_dim * 2**i) for i in range(len(config.depths))] # Add layer norms to hidden states of out_features From c7781fe4dee24feeb4c430face7f95d4dee91757 Mon Sep 17 00:00:00 2001 From: Eli Simhayev Date: Mon, 3 Apr 2023 20:07:21 +0700 Subject: [PATCH 246/277] [Time-Series] fix past_observed_mask type (#22076) added > 0.5 to `past_observed_mask` --- tests/models/informer/test_modeling_informer.py | 2 +- .../test_modeling_time_series_transformer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/informer/test_modeling_informer.py b/tests/models/informer/test_modeling_informer.py index 91e6fb74f6f1..271f997bee8b 100644 --- a/tests/models/informer/test_modeling_informer.py +++ b/tests/models/informer/test_modeling_informer.py @@ -117,7 +117,7 @@ def prepare_informer_inputs_dict(self, config): past_time_features = floats_tensor([self.batch_size, _past_length, config.num_time_features]) past_values = floats_tensor([self.batch_size, _past_length]) - past_observed_mask = floats_tensor([self.batch_size, _past_length]) + past_observed_mask = floats_tensor([self.batch_size, _past_length]) > 0.5 # decoder inputs future_time_features = floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]) diff --git a/tests/models/time_series_transformer/test_modeling_time_series_transformer.py b/tests/models/time_series_transformer/test_modeling_time_series_transformer.py index 7f14f29de040..65834dac42f7 100644 --- a/tests/models/time_series_transformer/test_modeling_time_series_transformer.py +++ b/tests/models/time_series_transformer/test_modeling_time_series_transformer.py @@ -114,7 +114,7 @@ def prepare_time_series_transformer_inputs_dict(self, config): past_time_features = floats_tensor([self.batch_size, _past_length, config.num_time_features]) past_values = floats_tensor([self.batch_size, _past_length]) - past_observed_mask = floats_tensor([self.batch_size, _past_length]) + past_observed_mask = floats_tensor([self.batch_size, _past_length]) > 0.5 # decoder inputs future_time_features = floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]) From a2e3570faddaa3fd630e5e3fdbf903580f86784a Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Mon, 3 Apr 2023 15:07:32 +0200 Subject: [PATCH 247/277] Fix llama tokenizer (#22402) * draft * update tokenization limma and conversion script * more udpates * initial commit * style * default pad to None * draft tokenization tests * update test * update tokenization tests * nits * update * versioning test * major fix * fix more testst * finish fixing special masks * last nit * more nits * add encode decode tests * add more * fix token type ids * style --- .../llama/convert_llama_weights_to_hf.py | 18 +- .../models/llama/tokenization_llama.py | 110 +++-- tests/models/llama/test_tokenization_llama.py | 412 ++++++++++++++++++ tests/test_tokenization_common.py | 2 +- 4 files changed, 480 insertions(+), 62 deletions(-) create mode 100644 tests/models/llama/test_tokenization_llama.py diff --git a/src/transformers/models/llama/convert_llama_weights_to_hf.py b/src/transformers/models/llama/convert_llama_weights_to_hf.py index 3ba48f7c6fe4..3dc6c7d69700 100644 --- a/src/transformers/models/llama/convert_llama_weights_to_hf.py +++ b/src/transformers/models/llama/convert_llama_weights_to_hf.py @@ -20,7 +20,7 @@ import torch -from transformers import LlamaConfig, LlamaForCausalLM +from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer """ @@ -233,19 +233,9 @@ def permute(w): def write_tokenizer(tokenizer_path, input_tokenizer_path): print(f"Fetching the tokenizer from {input_tokenizer_path}.") - os.makedirs(tokenizer_path, exist_ok=True) - write_json({}, os.path.join(tokenizer_path, "special_tokens_map.json")) - write_json( - { - "bos_token": "", - "eos_token": "", - "model_max_length": int(1e30), - "tokenizer_class": "LlamaTokenizer", - "unk_token": "", - }, - os.path.join(tokenizer_path, "tokenizer_config.json"), - ) - shutil.copyfile(input_tokenizer_path, os.path.join(tokenizer_path, "tokenizer.model")) + # Initialize the tokenizer based on the `spm` model + tokenizer = LlamaTokenizer(input_tokenizer_path) + tokenizer.save_pretrained(tokenizer_path) def main(): diff --git a/src/transformers/models/llama/tokenization_llama.py b/src/transformers/models/llama/tokenization_llama.py index 618af846cea7..d6daa1006436 100644 --- a/src/transformers/models/llama/tokenization_llama.py +++ b/src/transformers/models/llama/tokenization_llama.py @@ -25,7 +25,7 @@ import sentencepiece as spm -from ...tokenization_utils import PreTrainedTokenizer +from ...tokenization_utils import AddedToken, PreTrainedTokenizer from ...utils import logging @@ -33,7 +33,17 @@ VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"} -PRETRAINED_VOCAB_FILES_MAP = {} +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "hf-internal-testing/llama-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model", + }, + "tokenizer_file": { + "hf-internal-testing/llama-tokenizer": "https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer_config.json", + }, +} +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "hf-internal-testing/llama-tokenizer": 2048, +} class LlamaTokenizer(PreTrainedTokenizer): @@ -47,6 +57,7 @@ class LlamaTokenizer(PreTrainedTokenizer): vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES model_input_names = ["input_ids", "attention_mask"] def __init__( @@ -55,51 +66,50 @@ def __init__( unk_token="", bos_token="", eos_token="", + pad_token=None, sp_model_kwargs: Optional[Dict[str, Any]] = None, add_bos_token=True, add_eos_token=False, - decode_with_prefix_space=False, clean_up_tokenization_spaces=False, **kwargs, ): self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token + eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token + unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token + pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token super().__init__( bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, + pad_token=pad_token, + add_bos_token=add_bos_token, + add_eos_token=add_eos_token, + sp_model_kwargs=self.sp_model_kwargs, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs, ) self.vocab_file = vocab_file self.add_bos_token = add_bos_token self.add_eos_token = add_eos_token - self.decode_with_prefix_space = decode_with_prefix_space self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) self.sp_model.Load(vocab_file) - self._no_prefix_space_tokens = None - """ Initialisation""" + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state - @property - def no_prefix_space_tokens(self): - if self._no_prefix_space_tokens is None: - vocab = self.convert_ids_to_tokens(list(range(self.vocab_size))) - self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith("▁")} - return self._no_prefix_space_tokens + def __setstate__(self, d): + self.__dict__ = d + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(self.vocab_file) @property def vocab_size(self): """Returns vocab size""" return self.sp_model.get_piece_size() - @property - def bos_token_id(self) -> Optional[int]: - return self.sp_model.bos_id() - - @property - def eos_token_id(self) -> Optional[int]: - return self.sp_model.eos_id() - def get_vocab(self): """Returns vocab as a dict""" vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} @@ -119,21 +129,15 @@ def _convert_id_to_token(self, index): token = self.sp_model.IdToPiece(index) return token - def _maybe_add_prefix_space(self, tokens, decoded): - if tokens and tokens[0] not in self.no_prefix_space_tokens: - return " " + decoded - else: - return decoded - def convert_tokens_to_string(self, tokens): """Converts a sequence of tokens (string) in a single string.""" current_sub_tokens = [] out_string = "" prev_is_special = False - for token in tokens: + for i, token in enumerate(tokens): # make sure that special tokens are not decoded using sentencepiece model if token in self.all_special_tokens: - if not prev_is_special: + if not prev_is_special and i != 0: out_string += " " out_string += self.sp_model.decode(current_sub_tokens) + token prev_is_special = True @@ -142,7 +146,6 @@ def convert_tokens_to_string(self, tokens): current_sub_tokens.append(token) prev_is_special = False out_string += self.sp_model.decode(current_sub_tokens) - out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string) return out_string def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]: @@ -173,18 +176,13 @@ def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) return (out_vocab_file,) def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): - if self.add_bos_token: - bos_token_ids = [self.bos_token_id] - else: - bos_token_ids = [] + bos_token_id = [self.bos_token_id] if self.add_bos_token else [] + eos_token_id = [self.eos_token_id] if self.add_eos_token else [] - output = bos_token_ids + token_ids_0 + output = bos_token_id + token_ids_0 + eos_token_id if token_ids_1 is not None: - output = output + token_ids_1 - - if self.add_eos_token: - output = output + [self.eos_token_id] + output = output + bos_token_id + token_ids_1 + eos_token_id return output @@ -211,28 +209,46 @@ def get_special_tokens_mask( token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True ) + bos_token_id = [1] if self.add_bos_token else [] + eos_token_id = [1] if self.add_eos_token else [] + if token_ids_1 is None: - return [1] + ([0] * len(token_ids_0)) + [1] - return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id + return ( + bos_token_id + + ([0] * len(token_ids_0)) + + eos_token_id + + bos_token_id + + ([0] * len(token_ids_1)) + + eos_token_id + ) def create_token_type_ids_from_sequences( self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None ) -> List[int]: """ - Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make - use of token type ids, therefore a list of zeros is returned. + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT + sequence pair mask has the following format: + + ``` + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + ``` + + if token_ids_1 is None, only returns the first portion of the mask (0s). Args: token_ids_0 (`List[int]`): - List of IDs. + List of ids. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: - `List[int]`: List of zeros. + `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). """ - eos = [self.eos_token_id] + sep = [self.sep_token_id] + cls = [self.cls_token_id] if token_ids_1 is None: - return len(token_ids_0 + eos) * [0] - return len(token_ids_0 + eos + token_ids_1 + eos) * [0] + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] diff --git a/tests/models/llama/test_tokenization_llama.py b/tests/models/llama/test_tokenization_llama.py new file mode 100644 index 000000000000..9950149c02c8 --- /dev/null +++ b/tests/models/llama/test_tokenization_llama.py @@ -0,0 +1,412 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile +import unittest + +from datasets import load_dataset + +from transformers import ( + SPIECE_UNDERLINE, + AddedToken, + LlamaTokenizer, + is_torch_available, +) +from transformers.testing_utils import ( + get_tests_dir, + nested_simplify, + require_sentencepiece, + require_tokenizers, + require_torch, + slow, +) + +from ...test_tokenization_common import TokenizerTesterMixin + + +SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model") + + +if is_torch_available(): + pass + + +@require_sentencepiece +@require_tokenizers +class LlamaTokenizationTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = LlamaTokenizer + test_rust_tokenizer = False + test_sentencepiece = True + from_pretrained_kwargs = {} + + def setUp(self): + super().setUp() + + # We have a SentencePiece fixture for testing + tokenizer = LlamaTokenizer(SAMPLE_VOCAB, keep_accents=True) + tokenizer.pad_token = tokenizer.eos_token + tokenizer.save_pretrained(self.tmpdirname) + + def test_full_tokenizer(self): + tokenizer = LlamaTokenizer(SAMPLE_VOCAB, keep_accents=True) + + tokens = tokenizer.tokenize("This is a test") + self.assertListEqual(tokens, ["▁This", "▁is", "▁a", "▁t", "est"]) + + self.assertListEqual( + tokenizer.convert_tokens_to_ids(tokens), + [285, 46, 10, 170, 382], + ) + + tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.") + self.assertListEqual( + tokens, + [ + SPIECE_UNDERLINE + "I", + SPIECE_UNDERLINE + "was", + SPIECE_UNDERLINE + "b", + "or", + "n", + SPIECE_UNDERLINE + "in", + SPIECE_UNDERLINE + "", + "9", + "2", + "0", + "0", + "0", + ",", + SPIECE_UNDERLINE + "and", + SPIECE_UNDERLINE + "this", + SPIECE_UNDERLINE + "is", + SPIECE_UNDERLINE + "f", + "al", + "s", + "é", + ".", + ], + ) + ids = tokenizer.convert_tokens_to_ids(tokens) + self.assertListEqual( + ids, + [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4], + ) + + back_tokens = tokenizer.convert_ids_to_tokens(ids) + self.assertListEqual( + back_tokens, + [ + SPIECE_UNDERLINE + "I", + SPIECE_UNDERLINE + "was", + SPIECE_UNDERLINE + "b", + "or", + "n", + SPIECE_UNDERLINE + "in", + SPIECE_UNDERLINE + "", + "", + "2", + "0", + "0", + "0", + ",", + SPIECE_UNDERLINE + "and", + SPIECE_UNDERLINE + "this", + SPIECE_UNDERLINE + "is", + SPIECE_UNDERLINE + "f", + "al", + "s", + "", + ".", + ], + ) + + @unittest.skip("Let's wait for the fast tokenizer!") + def test_save_pretrained(self): + self.tokenizers_list += (self.rust_tokenizer_class, "hf-internal-testing/llama-tokenizer", {}) + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: + with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): + tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) + tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs) + + tmpdirname2 = tempfile.mkdtemp() + + tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2) + tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2) + + # Checks it save with the same files + the tokenizer.json file for the fast one + self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files)) + tokenizer_r_files = tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f) + self.assertSequenceEqual(tokenizer_r_files, tokenizer_p_files) + + # Checks everything loads correctly in the same way + tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2) + tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2) + + # Check special tokens are set accordingly on Rust and Python + for key in tokenizer_pp.special_tokens_map: + self.assertTrue(hasattr(tokenizer_rp, key)) + + shutil.rmtree(tmpdirname2) + + # Save tokenizer rust, legacy_format=True + tmpdirname2 = tempfile.mkdtemp() + + tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2, legacy_format=True) + tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2) + + # Checks it save with the same files + self.assertSequenceEqual(tokenizer_r_files, tokenizer_p_files) + + # Checks everything loads correctly in the same way + tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2) + tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2) + + # Check special tokens are set accordingly on Rust and Python + for key in tokenizer_pp.special_tokens_map: + self.assertTrue(hasattr(tokenizer_rp, key)) + + shutil.rmtree(tmpdirname2) + + # Save tokenizer rust, legacy_format=False + tmpdirname2 = tempfile.mkdtemp() + + tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2, legacy_format=False) + tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2) + + # Checks it saved the tokenizer.json file + self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files)) + + # Checks everything loads correctly in the same way + tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2) + tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2) + + # Check special tokens are set accordingly on Rust and Python + for key in tokenizer_pp.special_tokens_map: + self.assertTrue(hasattr(tokenizer_rp, key)) + + shutil.rmtree(tmpdirname2) + + @require_torch + def test_batch_tokenization(self): + if not self.test_seq2seq: + return + + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + # Longer text that will definitely require truncation. + text = [ + " UN Chief Says There Is No Military Solution in Syria", + " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for" + " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons" + " will only worsen the violence and misery for millions of people.", + ] + try: + batch = tokenizer( + text=text, + max_length=3, + max_target_length=10, + return_tensors="pt", + ) + except NotImplementedError: + return + self.assertEqual(batch.input_ids.shape[1], 3) + # max_target_length will default to max_length if not specified + batch = tokenizer(text, max_length=3, return_tensors="pt") + self.assertEqual(batch.input_ids.shape[1], 3) + + batch_encoder_only = tokenizer(text=text, max_length=3, max_target_length=10, return_tensors="pt") + self.assertEqual(batch_encoder_only.input_ids.shape[1], 3) + self.assertEqual(batch_encoder_only.attention_mask.shape[1], 3) + self.assertNotIn("decoder_input_ids", batch_encoder_only) + + @unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece.") + def test_save_slow_from_fast_and_reload_fast(self): + pass + + def test_special_tokens_initialization(self): + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: + with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): + added_tokens = [AddedToken("", lstrip=True)] + + tokenizer_r = self.rust_tokenizer_class.from_pretrained( + pretrained_name, additional_special_tokens=added_tokens, **kwargs + ) + r_output = tokenizer_r.encode("Hey this is a token") + + special_token_id = tokenizer_r.encode("", add_special_tokens=False)[0] + + self.assertTrue(special_token_id in r_output) + + if self.test_slow_tokenizer: + tokenizer_cr = self.rust_tokenizer_class.from_pretrained( + pretrained_name, + additional_special_tokens=added_tokens, + **kwargs, # , from_slow=True <- unfortunately too slow to convert + ) + tokenizer_p = self.tokenizer_class.from_pretrained( + pretrained_name, additional_special_tokens=added_tokens, **kwargs + ) + + p_output = tokenizer_p.encode("Hey this is a token") + + cr_output = tokenizer_cr.encode("Hey this is a token") + + self.assertEqual(p_output, r_output) + self.assertEqual(cr_output, r_output) + self.assertTrue(special_token_id in p_output) + self.assertTrue(special_token_id in cr_output) + + @slow + def test_tokenizer_integration(self): + # fmt: off + expected_encoding = {'input_ids': [[1, 4103, 689, 414, 313, 24784, 368, 2998, 408, 282, 3637, 25350, 29899, 9067, 414, 322, 282, 3637, 25350, 29899, 1457, 3018, 1312, 29899, 2151, 29897, 8128, 2498, 29899, 15503, 4220, 6956, 1973, 313, 13635, 29911, 29892, 402, 7982, 29899, 29906, 29892, 1528, 13635, 29911, 29874, 29892, 1060, 26369, 29892, 6652, 309, 29933, 814, 29892, 1060, 29931, 6779, 11410, 363, 18385, 17088, 7634, 11235, 313, 25103, 29965, 29897, 322, 18385, 17088, 28203, 313, 25103, 29954, 29897, 411, 975, 29871, 29941, 29906, 29974, 758, 3018, 1312, 4733, 297, 29871, 29896, 29900, 29900, 29974, 10276, 322, 6483, 1006, 3372, 3097, 1546, 435, 1165, 29892, 10772, 29911, 25350, 322, 323, 6073, 17907, 29889], [1, 350, 20161, 338, 8688, 304, 758, 29899, 14968, 6483, 21000, 8684, 284, 22540, 515, 443, 29880, 24025, 1426, 491, 14002, 368, 4195, 292, 373, 1716, 2175, 322, 1492, 3030, 297, 599, 15359, 29889], [1, 450, 4996, 17354, 1701, 29916, 432, 17204, 975, 278, 17366, 11203, 29889]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]} + # fmt: on + + self.tokenizer_integration_test_util( + expected_encoding=expected_encoding, + model_name="hf-internal-testing/llama-tokenizer", + revision="0984d03108b1a041ed679bd253b6519b7e1a4778", + padding=False, + ) + + +@require_torch +@require_sentencepiece +@require_tokenizers +class LlamaIntegrationTest(unittest.TestCase): + checkpoint_name = "hf-internal-testing/llama-tokenizer" + + @classmethod + def setUpClass(cls): + cls.tokenizer: LlamaTokenizer = LlamaTokenizer.from_pretrained(cls.checkpoint_name) + cls.rust_tokenizer = cls.tokenizer # TODO @narsil replace with the rust one + cls.pad_token_id = 1 + return cls + + @require_torch + def integration_tests(self): + inputs = self.tokenizer( + ["The following string should be properly encoded: Hello.", "But ird and ปี ird ด"], + return_tensors="pt", + ) + + self.assertEqual( + nested_simplify(inputs), + { + "input_ids": [ + [1, 450, 1494, 1347, 881, 367, 6284, 18511, 29901, 15043, 29889], + [1, 1205, 29871, 1823, 322, 29871, 31010, 30691, 1678, 1823, 1678, 30718], + ], + "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], + }, + ) + + def test_simple_encode_decode(self): + pyth_tokenizer = self.tokenizer + rust_tokenizer = self.rust_tokenizer + + self.assertEqual(pyth_tokenizer.encode("This is a test"), [1, 910, 338, 263, 1243]) + self.assertEqual(rust_tokenizer.encode("This is a test"), [1, 910, 338, 263, 1243]) + self.assertEqual(pyth_tokenizer.decode([1, 910, 338, 263, 1243], skip_special_tokens=True), "This is a test") + self.assertEqual(rust_tokenizer.decode([1, 910, 338, 263, 1243], skip_special_tokens=True), "This is a test") + + # bytefallback showcase + self.assertEqual(pyth_tokenizer.encode("生活的真谛是"), [1, 29871, 30486, 31704, 30210, 30848, 235, 179, 158, 30392]) + self.assertEqual(rust_tokenizer.encode("生活的真谛是"), [1, 29871, 30486, 31704, 30210, 30848, 235, 179, 158, 30392]) + self.assertEqual( + pyth_tokenizer.decode( + [1, 29871, 30486, 31704, 30210, 30848, 235, 179, 158, 30392], skip_special_tokens=True + ), + "生活的真谛是", + ) + self.assertEqual( + rust_tokenizer.decode( + [1, 29871, 30486, 31704, 30210, 30848, 235, 179, 158, 30392], skip_special_tokens=True + ), + "生活的真谛是", + ) + + # Inner spaces showcase + self.assertEqual(pyth_tokenizer.encode("Hi Hello"), [1, 6324, 29871, 15043]) + self.assertEqual(rust_tokenizer.encode("Hi Hello"), [1, 6324, 29871, 15043]) + self.assertEqual(pyth_tokenizer.decode([1, 6324, 29871, 15043], skip_special_tokens=True), "Hi Hello") + self.assertEqual(rust_tokenizer.decode([1, 6324, 29871, 15043], skip_special_tokens=True), "Hi Hello") + + self.assertEqual(pyth_tokenizer.encode("Hi Hello"), [1, 6324, 259, 15043]) + self.assertEqual(rust_tokenizer.encode("Hi Hello"), [1, 6324, 259, 15043]) + self.assertEqual(pyth_tokenizer.decode([1, 6324, 259, 15043], skip_special_tokens=True), "Hi Hello") + self.assertEqual(rust_tokenizer.decode([1, 6324, 259, 15043], skip_special_tokens=True), "Hi Hello") + + self.assertEqual(pyth_tokenizer.encode(""), [1]) + self.assertEqual(rust_tokenizer.encode(""), [1]) + + self.assertEqual(pyth_tokenizer.encode(" "), [1, 259]) + self.assertEqual(rust_tokenizer.encode(" "), [1, 259]) + + self.assertEqual(pyth_tokenizer.encode(" "), [1, 1678]) + self.assertEqual(rust_tokenizer.encode(" "), [1, 1678]) + + self.assertEqual(pyth_tokenizer.encode(" Hello"), [1, 29871, 15043]) + self.assertEqual(rust_tokenizer.encode(" Hello"), [1, 29871, 15043]) + + self.assertEqual(pyth_tokenizer.encode(""), [1, 1]) + self.assertEqual(rust_tokenizer.encode(""), [1, 1]) + + self.assertEqual(pyth_tokenizer.encode(""), [1]) + self.assertEqual(rust_tokenizer.encode(""), [1]) + + self.assertEqual(pyth_tokenizer.decode([869]), ".") + self.assertEqual(rust_tokenizer.decode([869]), ".") + + self.assertEqual(pyth_tokenizer.decode([30112, 869]), "ا .") + self.assertEqual(rust_tokenizer.decode([30112, 869]), "ا .") + + @unittest.skipIf( + os.getenv("RUN_TOKENIZER_INTEGRATION", "0") == "0", + "RUN_TOKENIZER_INTEGRATION=1 to run tokenizer integration tests", + ) + def test_integration_test_xnli(self): + import tqdm + + pyth_tokenizer = self.tokenizer + rust_tokenizer = self.rust_tokenizer + + dataset = load_dataset("code_x_glue_ct_code_to_text", "go") + for item in tqdm.tqdm(dataset["validation"]): + string = item["code"] + encoded1 = pyth_tokenizer.encode(string) + encoded2 = rust_tokenizer.encode(string) + + self.assertEqual(encoded1, encoded2) + + decoded1 = pyth_tokenizer.decode(encoded1) + decoded2 = rust_tokenizer.decode(encoded2) + + self.assertEqual(decoded1, decoded2) + + dataset = load_dataset("xnli", "all_languages") + + for item in tqdm.tqdm(dataset["train"]): + for string in item["premise"].values(): + encoded1 = pyth_tokenizer.encode(string) + encoded2 = rust_tokenizer.encode(string) + + self.assertEqual(encoded1, encoded2) + + decoded1 = pyth_tokenizer.decode(encoded1) + decoded2 = rust_tokenizer.decode(encoded2) + + self.assertEqual(decoded1, decoded2) diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 53a12aedd6a9..836f66a14aa4 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -3932,7 +3932,7 @@ def test_clean_up_tokenization_spaces(self): tokenizer_fast.save_pretrained(tmp_dir_2) tokenizer = BertTokenizer.from_pretrained(tmp_dir_2) - assert tokenizer_fast.clean_up_tokenization_spaces is False + assert tokenizer.clean_up_tokenization_spaces is False decoded = tokenizer.decode(tokens) assert decoded == "[CLS] this shouldn ' t be ! he ' ll go . [SEP]" From aa1a3663dc9200670a5316653438b4659e817af8 Mon Sep 17 00:00:00 2001 From: Jungnerd <46880056+jungnerd@users.noreply.github.com> Date: Mon, 3 Apr 2023 22:17:02 +0900 Subject: [PATCH 248/277] [WIP] docs: ko: sagemaker.mdx (#22509) docs: ko: sagemaker.mdx --- docs/source/ko/_toctree.yml | 2 ++ docs/source/ko/sagemaker.mdx | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 docs/source/ko/sagemaker.mdx diff --git a/docs/source/ko/_toctree.yml b/docs/source/ko/_toctree.yml index 06ab5ee76634..8d9b16395e28 100644 --- a/docs/source/ko/_toctree.yml +++ b/docs/source/ko/_toctree.yml @@ -15,6 +15,8 @@ title: (번역 중) title: How-to 가이드 - sections: + - local: sagemaker + title: Amazon SageMaker에서 학습 실행하기 - local: in_translation title: (번역 중) title: 개념 가이드 diff --git a/docs/source/ko/sagemaker.mdx b/docs/source/ko/sagemaker.mdx new file mode 100644 index 000000000000..e52ca98e1d82 --- /dev/null +++ b/docs/source/ko/sagemaker.mdx @@ -0,0 +1,25 @@ + + +# Amazon SageMaker에서 학습 실행하기 + +문서가 [hf.co/docs/sagemaker](https://huggingface.co/docs/sagemaker)로 이동되었습니다. 이 페이지는 `transformers` 5.0 에서 삭제될 예정입니다. + +### 목차 + +- [Train Hugging Face models on Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/train) +- [Deploy Hugging Face models to Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/inference) +- [Frequently Asked Questions](https://huggingface.co/docs/sagemaker/faq) From fb0965242eae21814ac52cc18a8f3f30b3c3f070 Mon Sep 17 00:00:00 2001 From: Mohammed Jabir <40956091+upjabir@users.noreply.github.com> Date: Mon, 3 Apr 2023 18:50:02 +0530 Subject: [PATCH 249/277] added biogpt token classifier (#22447) * added biogpt token classifier * fix reviews * Updated modeling_biogpt.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --------- Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- docs/source/en/model_doc/biogpt.mdx | 5 + docs/source/en/tasks/token_classification.mdx | 2 +- src/transformers/__init__.py | 2 + src/transformers/models/auto/modeling_auto.py | 1 + src/transformers/models/biogpt/__init__.py | 2 + .../models/biogpt/modeling_biogpt.py | 98 ++++++++++++++++++- src/transformers/utils/dummy_pt_objects.py | 7 ++ tests/models/biogpt/test_modeling_biogpt.py | 26 ++++- 8 files changed, 138 insertions(+), 5 deletions(-) diff --git a/docs/source/en/model_doc/biogpt.mdx b/docs/source/en/model_doc/biogpt.mdx index e852c83a3645..b334a05ede68 100644 --- a/docs/source/en/model_doc/biogpt.mdx +++ b/docs/source/en/model_doc/biogpt.mdx @@ -53,4 +53,9 @@ This model was contributed by [kamalkraj](https://huggingface.co/kamalkraj). The ## BioGptForCausalLM [[autodoc]] BioGptForCausalLM + - forward + +## BioGptForTokenClassification + +[[autodoc]] BioGptForTokenClassification - forward \ No newline at end of file diff --git a/docs/source/en/tasks/token_classification.mdx b/docs/source/en/tasks/token_classification.mdx index 930361a354cd..c3013283d7d0 100644 --- a/docs/source/en/tasks/token_classification.mdx +++ b/docs/source/en/tasks/token_classification.mdx @@ -28,7 +28,7 @@ The task illustrated in this tutorial is supported by the following model archit -[ALBERT](../model_doc/albert), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LiLT](../model_doc/lilt), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [QDQBert](../model_doc/qdqbert), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) +[ALBERT](../model_doc/albert), [BERT](../model_doc/bert), [BigBird](../model_doc/big_bird), [BioGpt](../model_doc/biogpt), [BLOOM](../model_doc/bloom), [CamemBERT](../model_doc/camembert), [CANINE](../model_doc/canine), [ConvBERT](../model_doc/convbert), [Data2VecText](../model_doc/data2vec-text), [DeBERTa](../model_doc/deberta), [DeBERTa-v2](../model_doc/deberta-v2), [DistilBERT](../model_doc/distilbert), [ELECTRA](../model_doc/electra), [ERNIE](../model_doc/ernie), [ErnieM](../model_doc/ernie_m), [ESM](../model_doc/esm), [FlauBERT](../model_doc/flaubert), [FNet](../model_doc/fnet), [Funnel Transformer](../model_doc/funnel), [GPT-Sw3](../model_doc/gpt-sw3), [OpenAI GPT-2](../model_doc/gpt2), [I-BERT](../model_doc/ibert), [LayoutLM](../model_doc/layoutlm), [LayoutLMv2](../model_doc/layoutlmv2), [LayoutLMv3](../model_doc/layoutlmv3), [LiLT](../model_doc/lilt), [Longformer](../model_doc/longformer), [LUKE](../model_doc/luke), [MarkupLM](../model_doc/markuplm), [MEGA](../model_doc/mega), [Megatron-BERT](../model_doc/megatron-bert), [MobileBERT](../model_doc/mobilebert), [MPNet](../model_doc/mpnet), [Nezha](../model_doc/nezha), [Nyströmformer](../model_doc/nystromformer), [QDQBert](../model_doc/qdqbert), [RemBERT](../model_doc/rembert), [RoBERTa](../model_doc/roberta), [RoBERTa-PreLayerNorm](../model_doc/roberta-prelayernorm), [RoCBert](../model_doc/roc_bert), [RoFormer](../model_doc/roformer), [SqueezeBERT](../model_doc/squeezebert), [XLM](../model_doc/xlm), [XLM-RoBERTa](../model_doc/xlm-roberta), [XLM-RoBERTa-XL](../model_doc/xlm-roberta-xl), [XLNet](../model_doc/xlnet), [X-MOD](../model_doc/xmod), [YOSO](../model_doc/yoso) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 518c6524d334..ed69a5be194b 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1131,6 +1131,7 @@ [ "BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST", "BioGptForCausalLM", + "BioGptForTokenClassification", "BioGptModel", "BioGptPreTrainedModel", ] @@ -4704,6 +4705,7 @@ from .models.biogpt import ( BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST, BioGptForCausalLM, + BioGptForTokenClassification, BioGptModel, BioGptPreTrainedModel, ) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index f24146199876..c6eb89327941 100755 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -782,6 +782,7 @@ ("albert", "AlbertForTokenClassification"), ("bert", "BertForTokenClassification"), ("big_bird", "BigBirdForTokenClassification"), + ("biogpt", "BioGptForTokenClassification"), ("bloom", "BloomForTokenClassification"), ("camembert", "CamembertForTokenClassification"), ("canine", "CanineForTokenClassification"), diff --git a/src/transformers/models/biogpt/__init__.py b/src/transformers/models/biogpt/__init__.py index 64fd23977a5c..761b904013c2 100644 --- a/src/transformers/models/biogpt/__init__.py +++ b/src/transformers/models/biogpt/__init__.py @@ -30,6 +30,7 @@ _import_structure["modeling_biogpt"] = [ "BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST", "BioGptForCausalLM", + "BioGptForTokenClassification", "BioGptModel", "BioGptPreTrainedModel", ] @@ -48,6 +49,7 @@ from .modeling_biogpt import ( BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST, BioGptForCausalLM, + BioGptForTokenClassification, BioGptModel, BioGptPreTrainedModel, ) diff --git a/src/transformers/models/biogpt/modeling_biogpt.py b/src/transformers/models/biogpt/modeling_biogpt.py index 178e67f8e1bc..59b7f1f4ac7e 100755 --- a/src/transformers/models/biogpt/modeling_biogpt.py +++ b/src/transformers/models/biogpt/modeling_biogpt.py @@ -25,7 +25,11 @@ from torch.nn import CrossEntropyLoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions +from ...modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + CausalLMOutputWithCrossAttentions, + TokenClassifierOutput, +) from ...modeling_utils import PreTrainedModel from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_biogpt import BioGptConfig @@ -736,3 +740,95 @@ def _reorder_cache(past_key_values, beam_idx): for layer_past in past_key_values: reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) return reordered_past + + +@add_start_docstrings( + """ + BioGPT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for + Named-Entity-Recognition (NER) tasks. + """, + BIOGPT_START_DOCSTRING, +) +class BioGptForTokenClassification(BioGptPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.biogpt = BioGptModel(config) + if hasattr(config, "classifier_dropout") and config.classifier_dropout is not None: + classifier_dropout = config.classifier_dropout + else: + classifier_dropout = config.hidden_dropout_prob + self.dropout = nn.Dropout(classifier_dropout) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.post_init() + + @add_start_docstrings_to_model_forward(BIOGPT_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TokenClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, TokenClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.biogpt( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = transformer_outputs[0] + hidden_states = self.dropout(hidden_states) + logits = self.classifier(hidden_states) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels) + active_labels = torch.where( + active_loss, labels.view(-1), torch.tensor(loss_fct.ignore_index).type_as(labels) + ) + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + transformer_outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 4859974f22b2..6b246a8463f7 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1093,6 +1093,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class BioGptForTokenClassification(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class BioGptModel(metaclass=DummyObject): _backends = ["torch"] diff --git a/tests/models/biogpt/test_modeling_biogpt.py b/tests/models/biogpt/test_modeling_biogpt.py index ce6464b66098..4ad2edfc13f4 100644 --- a/tests/models/biogpt/test_modeling_biogpt.py +++ b/tests/models/biogpt/test_modeling_biogpt.py @@ -29,7 +29,7 @@ if is_torch_available(): import torch - from transformers import BioGptForCausalLM, BioGptModel, BioGptTokenizer + from transformers import BioGptForCausalLM, BioGptForTokenClassification, BioGptModel, BioGptTokenizer from transformers.models.biogpt.modeling_biogpt import BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST @@ -247,6 +247,16 @@ def create_and_check_biogpt_weight_initialization(self, config, *args): self.parent.assertLessEqual(abs(torch.std(model.state_dict()[key]) - model_std), 0.001) self.parent.assertLessEqual(abs(torch.mean(model.state_dict()[key]) - 0.0), 0.01) + def create_and_check_biogpt_for_token_classification( + self, config, input_ids, input_mask, head_mask, token_type_ids, *args + ): + config.num_labels = self.num_labels + model = BioGptForTokenClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels)) + def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() ( @@ -264,10 +274,16 @@ def prepare_config_and_inputs_for_common(self): @require_torch class BioGptModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase): - all_model_classes = (BioGptModel, BioGptForCausalLM) if is_torch_available() else () + all_model_classes = (BioGptModel, BioGptForCausalLM, BioGptForTokenClassification) if is_torch_available() else () all_generative_model_classes = (BioGptForCausalLM,) if is_torch_available() else () pipeline_model_mapping = ( - {"feature-extraction": BioGptModel, "text-generation": BioGptForCausalLM} if is_torch_available() else {} + { + "feature-extraction": BioGptModel, + "text-generation": BioGptForCausalLM, + "token-classification": BioGptForTokenClassification, + } + if is_torch_available() + else {} ) test_pruning = False @@ -304,6 +320,10 @@ def test_biogpt_weight_initialization(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_biogpt_weight_initialization(*config_and_inputs) + def test_biogpt_token_classification_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_biogpt_for_token_classification(*config_and_inputs) + @slow def test_batch_generation(self): model = BioGptForCausalLM.from_pretrained("microsoft/biogpt") From bbad89b19e31ce6cf31983879451719b308064e1 Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Mon, 3 Apr 2023 15:04:37 +0100 Subject: [PATCH 250/277] Generate: `TextIteratorStreamer` (streamer for gradio) (#22501) * haha text go brrr (but in gradio) --- docs/source/en/internal/generation_utils.mdx | 2 + src/transformers/__init__.py | 4 +- src/transformers/generation/__init__.py | 7 +- src/transformers/generation/streamers.py | 91 ++++++++++++++++++++ tests/generation/test_streamers.py | 29 ++++++- 5 files changed, 125 insertions(+), 8 deletions(-) diff --git a/docs/source/en/internal/generation_utils.mdx b/docs/source/en/internal/generation_utils.mdx index dd93c79e92bf..10b050b3f8b2 100644 --- a/docs/source/en/internal/generation_utils.mdx +++ b/docs/source/en/internal/generation_utils.mdx @@ -269,3 +269,5 @@ A [`Constraint`] can be used to force the generation to include specific tokens ## Streamers [[autodoc]] TextStreamer + +[[autodoc]] TextIteratorStreamer diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index ed69a5be194b..cbe2bb3c5ff1 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -96,7 +96,7 @@ "feature_extraction_sequence_utils": ["SequenceFeatureExtractor"], "feature_extraction_utils": ["BatchFeature", "FeatureExtractionMixin"], "file_utils": [], - "generation": ["GenerationConfig", "TextStreamer"], + "generation": ["GenerationConfig", "TextIteratorStreamer", "TextStreamer"], "hf_argparser": ["HfArgumentParser"], "image_transforms": [], "integrations": [ @@ -3771,7 +3771,7 @@ from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin # Generation - from .generation import GenerationConfig, TextStreamer + from .generation import GenerationConfig, TextIteratorStreamer, TextStreamer from .hf_argparser import HfArgumentParser # Integrations diff --git a/src/transformers/generation/__init__.py b/src/transformers/generation/__init__.py index d163c44dc7f0..bf87b6e5ff5f 100644 --- a/src/transformers/generation/__init__.py +++ b/src/transformers/generation/__init__.py @@ -17,7 +17,10 @@ from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available -_import_structure = {"configuration_utils": ["GenerationConfig"], "streamers": ["TextStreamer"]} +_import_structure = { + "configuration_utils": ["GenerationConfig"], + "streamers": ["TextIteratorStreamer", "TextStreamer"], +} try: if not is_torch_available(): @@ -149,7 +152,7 @@ if TYPE_CHECKING: from .configuration_utils import GenerationConfig - from .streamers import TextStreamer + from .streamers import TextIteratorStreamer, TextStreamer try: if not is_torch_available(): diff --git a/src/transformers/generation/streamers.py b/src/transformers/generation/streamers.py index d110693b0eac..78d98666b3d1 100644 --- a/src/transformers/generation/streamers.py +++ b/src/transformers/generation/streamers.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from queue import Queue from typing import TYPE_CHECKING @@ -102,3 +103,93 @@ def end(self): # Print a newline (and the remaining text, if any) print(printable_text, flush=True) + + +class TextIteratorStreamer(BaseStreamer): + """ + Streamer that stores print-ready text in a queue, to be used by a downstream application as an iterator. This is + useful for applications that want to use the generated text in a non-blocking way (e.g. in an interactive Gradio + demo). + + Parameters: + tokenizer (`AutoTokenizer`): + The tokenized used to decode the tokens. + + Examples: + + ```python + >>> from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer + >>> from threading import Thread + + >>> tok = AutoTokenizer.from_pretrained("gpt2") + >>> model = AutoModelForCausalLM.from_pretrained("gpt2") + >>> inputs = tok(["An increasing sequence: one,"], return_tensors="pt") + >>> streamer = TextIteratorStreamer(tok) + + >>> # Run the generation in a separate thread, so that we can fetch the generated text in a non-blocking way. + >>> generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=20) + >>> thread = Thread(target=model.generate, kwargs=generation_kwargs) + >>> thread.start() + >>> generated_text = "" + >>> for new_text in streamer: + ... generated_text += new_text + >>> generated_text + 'An increasing sequence: one, two, three, four, five, six, seven, eight, nine, ten, eleven,' + ``` + """ + + def __init__(self, tokenizer: "AutoTokenizer"): + self.tokenizer = tokenizer + self.token_cache = [] + self.print_len = 0 + self.queue = Queue() + self.stop_signal = None + + def __iter__(self): + return self + + def __next__(self): + value = self.queue.get() + if value == self.stop_signal: + raise StopIteration() + else: + return value + + def put(self, value): + """ + Recives tokens, decodes them, and pushes text to the queue as soon as it form entire words. + """ + if len(value.shape) > 1 and value.shape[0] > 1: + raise ValueError("TextStreamer only supports batch size 1") + elif len(value.shape) > 1: + value = value[0] + + # Add the new token to the cache and decodes the entire thing. + self.token_cache.extend(value.tolist()) + text = self.tokenizer.decode(self.token_cache) + + # After symbol for a new line, we flush the cache. + if text.endswith("\n"): + printable_text = text[self.print_len :] + self.token_cache = [] + self.print_len = 0 + # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, + # which may change with the subsequent token -- there are probably smarter ways to do this!) + else: + printable_text = text[self.print_len : text.rfind(" ") + 1] + self.print_len += len(printable_text) + self.queue.put(printable_text) + + def end(self): + """Flushes any remaining cache and puts the stop signal in the queue.""" + # Flush the cache, if it exists + if len(self.token_cache) > 0: + text = self.tokenizer.decode(self.token_cache) + printable_text = text[self.print_len :] + self.token_cache = [] + self.print_len = 0 + else: + printable_text = "" + + self.queue.put(printable_text) + self.queue.put(self.stop_signal) # Put the stop signal diff --git a/tests/generation/test_streamers.py b/tests/generation/test_streamers.py index 120623285904..bf465ab31c6b 100644 --- a/tests/generation/test_streamers.py +++ b/tests/generation/test_streamers.py @@ -14,8 +14,9 @@ # limitations under the License. import unittest +from threading import Thread -from transformers import AutoTokenizer, TextStreamer, is_torch_available +from transformers import AutoTokenizer, TextIteratorStreamer, TextStreamer, is_torch_available from transformers.testing_utils import CaptureStdout, require_torch, torch_device from ..test_modeling_common import ids_tensor @@ -27,7 +28,7 @@ @require_torch class StreamerTester(unittest.TestCase): - def test_text_streamer_stdout(self): + def test_text_streamer_matches_non_streaming(self): tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") model = AutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2").to(torch_device) model.config.eos_token_id = -1 @@ -39,6 +40,26 @@ def test_text_streamer_stdout(self): with CaptureStdout() as cs: streamer = TextStreamer(tokenizer) model.generate(input_ids, max_new_tokens=10, do_sample=False, streamer=streamer) - # The greedy text should be printed to stdout, except for the final "\n" in the streamer - self.assertEqual(cs.out[:-1], greedy_text) + streamer_text = cs.out[:-1] + + self.assertEqual(streamer_text, greedy_text) + + def test_iterator_streamer_matches_non_streaming(self): + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") + model = AutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2").to(torch_device) + model.config.eos_token_id = -1 + + input_ids = ids_tensor((1, 5), vocab_size=model.config.vocab_size).to(torch_device) + greedy_ids = model.generate(input_ids, max_new_tokens=10, do_sample=False) + greedy_text = tokenizer.decode(greedy_ids[0]) + + streamer = TextIteratorStreamer(tokenizer) + generation_kwargs = {"input_ids": input_ids, "max_new_tokens": 10, "do_sample": False, "streamer": streamer} + thread = Thread(target=model.generate, kwargs=generation_kwargs) + thread.start() + streamer_text = "" + for new_text in streamer: + streamer_text += new_text + + self.assertEqual(streamer_text, greedy_text) From a08de03d682aedac747a6b8ef4ddda808a3265fb Mon Sep 17 00:00:00 2001 From: larekrow <127832774+larekrow@users.noreply.github.com> Date: Mon, 3 Apr 2023 22:06:52 +0800 Subject: [PATCH 251/277] Fix convert_opt_original_pytorch_checkpoint_to_pytorch.py typo (#22526) `load_checkpoint()` silently fails because `".qkj_proj." in key` is always `False`, but will eventually cause an error at `model.load_state_dict(state_dict)`. --- .../opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index 2a84641ce072..3f302b2ec3f4 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -55,9 +55,9 @@ def load_checkpoint(checkpoint_path): keys = list(sd.keys()) for key in keys: - if ".qkj_proj." in key: + if ".qkv_proj." in key: value = sd[key] - # We split QKV in seperate Q,K,V + # We split QKV in separate Q,K,V q_name = key.replace(".qkv_proj.", ".q_proj.") k_name = key.replace(".qkv_proj.", ".k_proj.") From af961bf925e6e1cabe062b6f6b65f78d8f305cf3 Mon Sep 17 00:00:00 2001 From: Kirill Date: Mon, 3 Apr 2023 17:28:40 +0300 Subject: [PATCH 252/277] llama docs: fix conversion script url (#22514) --- docs/source/en/model_doc/llama.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/llama.mdx b/docs/source/en/model_doc/llama.mdx index 158ee2b1aa3d..4acc868cc30a 100644 --- a/docs/source/en/model_doc/llama.mdx +++ b/docs/source/en/model_doc/llama.mdx @@ -23,7 +23,7 @@ The abstract from the paper is the following: Tips: - Weights for the LLaMA models can be obtained from by filling out [this form](https://docs.google.com/forms/d/e/1FAIpQLSfqNECQnMkycAp2jP4Z9TFX0cGR4uf7b_fBxjY_OjhJILlKGA/viewform?usp=send_form) -- After downloading the weights, they will need to be converted to the Hugging Face Transformers format using the [conversion script](/src/transformers/models/llama/convert_llama_weights_to_hf.py). The script can be called with the following (example) command: +- After downloading the weights, they will need to be converted to the Hugging Face Transformers format using the [conversion script](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/convert_llama_weights_to_hf.py). The script can be called with the following (example) command: ```bash python src/transformers/models/llama/convert_llama_weights_to_hf.py \ @@ -73,4 +73,4 @@ This model was contributed by [zphang](https://huggingface.co/zphang) with contr ## LlamaForSequenceClassification [[autodoc]] LlamaForSequenceClassification - - forward \ No newline at end of file + - forward From 5f2e5cd4a405b792633b613155a0694603807d14 Mon Sep 17 00:00:00 2001 From: Thibault Douzon Date: Mon, 3 Apr 2023 16:32:36 +0200 Subject: [PATCH 253/277] =?UTF-8?q?fix=20LayoutLMv3TokenizerFast=20subword?= =?UTF-8?q?=20label=20after=20'=C4=A0'=20token=20(#21695)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LayoutLMv3TokenizerFast produces empty 'Ġ' token with `offset_mapping = (0, 0)`. Next token is wrongly assumed to also be beginning of word and isn't correctly assigned `pad_token_label`. Modify test with text that produce 'Ġ' token. Remove copy check from LayoutLMv2TokenizerFast for `_batch_encode_plus`. solves issue: #19978 --- .../layoutlmv3/tokenization_layoutlmv3_fast.py | 8 ++++++-- .../layoutlmv3/test_tokenization_layoutlmv3.py | 12 ++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py b/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py index 4bd3e91480c0..3d7445e44931 100644 --- a/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py +++ b/src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py @@ -508,7 +508,6 @@ def encode_plus( **kwargs, ) - # Copied from transformers.models.layoutlmv2.tokenization_layoutlmv2_fast.LayoutLMv2TokenizerFast._batch_encode_plus with LayoutLMv2->LayoutLMv3 def _batch_encode_plus( self, batch_text_or_text_pairs: Union[ @@ -640,6 +639,7 @@ def _batch_encode_plus( else: original_index = batch_index labels_example = [] + previous_token_empty = False for id, offset, word_id in zip( sanitized_tokens["input_ids"][batch_index], sanitized_tokens["offset_mapping"][batch_index], @@ -647,11 +647,15 @@ def _batch_encode_plus( ): if word_id is not None: if self.only_label_first_subword: - if offset[0] == 0: + if offset[0] == 0 and not previous_token_empty: # Use the real label id for the first token of the word, and padding ids for the remaining tokens labels_example.append(word_labels[original_index][word_id]) else: labels_example.append(self.pad_token_label) + if offset == (0, 0): + previous_token_empty = True + else: + previous_token_empty = False else: labels_example.append(word_labels[original_index][word_id]) else: diff --git a/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py b/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py index 884f87680353..63d86f280cc0 100644 --- a/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py +++ b/tests/models/layoutlmv3/test_tokenization_layoutlmv3.py @@ -2277,14 +2277,14 @@ def test_compare_prepare_for_model(self): @slow def test_only_label_first_subword(self): - words = ["hello", "niels"] + words = ["hello", "niels", "0000000000000000"] boxes = [[1000, 1000, 1000, 1000] for _ in range(len(words))] - word_labels = [0, 1] + word_labels = [0, 1, 2] # test slow tokenizer tokenizer_p = LayoutLMv3Tokenizer.from_pretrained("microsoft/layoutlmv3-base", add_visual_labels=False) encoding = tokenizer_p(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, -100, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, -100, 2, -100, -100]) tokenizer_p = LayoutLMv3Tokenizer.from_pretrained( "microsoft/layoutlmv3-base", @@ -2292,12 +2292,12 @@ def test_only_label_first_subword(self): add_visual_labels=False, ) encoding = tokenizer_p(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, 1, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, 1, 2, 2, -100]) # test fast tokenizer tokenizer_r = LayoutLMv3TokenizerFast.from_pretrained("microsoft/layoutlmv3-base", add_visual_labels=False) encoding = tokenizer_r(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, -100, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, -100, 2, -100, -100]) tokenizer_r = LayoutLMv3Tokenizer.from_pretrained( "microsoft/layoutlmv3-base", @@ -2305,7 +2305,7 @@ def test_only_label_first_subword(self): add_visual_labels=False, ) encoding = tokenizer_r(words, boxes=boxes, word_labels=word_labels) - self.assertListEqual(encoding.labels, [-100, 0, 1, 1, -100]) + self.assertListEqual(encoding.labels, [-100, 0, 1, 1, 2, 2, -100]) @slow def test_layoutlmv3_integration_test(self): From fe24b030b99ed82d78d1069c77429ce9d2db499e Mon Sep 17 00:00:00 2001 From: zhbh01 Date: Mon, 3 Apr 2023 23:00:26 +0800 Subject: [PATCH 254/277] [BLIP] fix cross attentions for BlipTextEncoder (#22515) --- src/transformers/models/blip/modeling_blip_text.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 62351293ddaf..b600c2140eef 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -445,6 +445,7 @@ def custom_forward(*inputs): next_decoder_cache += (layer_outputs[-1],) if output_attentions: all_self_attentions = all_self_attentions + (layer_outputs[1],) + all_cross_attentions = all_cross_attentions + (layer_outputs[2],) if output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) From 0d419e1dc74660f8afe4f5647c6f8be4500e8601 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:10:50 +0200 Subject: [PATCH 255/277] [`Trainer`] Force `is_model_parallel` when model is loaded in multiple GPUs using `accelerate` (#22532) * add `is_model_parallel` arg on Trainer * add warning * adapt from suggestions * revert t5 changes * remove commas * adapt from suggestions --- src/transformers/trainer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index a41d43edeb4f..615836e11604 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -370,6 +370,19 @@ def __init__( else: self.is_model_parallel = False + if ( + getattr(model, "hf_device_map", None) is not None + and len([device for device in set(model.hf_device_map.values()) if device not in ["cpu", "disk"]]) > 1 + and not self.is_model_parallel + ): + self.is_model_parallel = True + + # warn users + logger.info( + "You have loaded a model on multiple GPUs. `is_model_parallel` attribute will be force-set" + " to `True` to avoid any unexpected behavior such as device placement mismatching." + ) + # At this stage the model is already loaded if getattr(model, "is_loaded_in_8bit", False): if getattr(model, "_is_int8_training_enabled", False): From 1376d600d62a79987b56129eb7d4f3c8590e49e8 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:55:37 +0200 Subject: [PATCH 256/277] [`T5`] Enable naive Pipeline Parallelism training for T5 (#22535) * enable PP for T5 * make fixup * fix failing tests --- src/transformers/models/mt5/modeling_mt5.py | 2 ++ src/transformers/models/t5/modeling_t5.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/transformers/models/mt5/modeling_mt5.py b/src/transformers/models/mt5/modeling_mt5.py index 43beb69f10f2..7e7a8903c011 100644 --- a/src/transformers/models/mt5/modeling_mt5.py +++ b/src/transformers/models/mt5/modeling_mt5.py @@ -1778,6 +1778,8 @@ def forward( loss = None if labels is not None: loss_fct = CrossEntropyLoss(ignore_index=-100) + # move labels to correct device to enable PP + labels = labels.to(lm_logits.device) loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1)) # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py index c056ef73cc1f..dfb9907a3cf5 100644 --- a/src/transformers/models/t5/modeling_t5.py +++ b/src/transformers/models/t5/modeling_t5.py @@ -1746,6 +1746,8 @@ def forward( loss = None if labels is not None: loss_fct = CrossEntropyLoss(ignore_index=-100) + # move labels to correct device to enable PP + labels = labels.to(lm_logits.device) loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1)) # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 From 7adc1632c4a91127657db6c53f0209448ddef962 Mon Sep 17 00:00:00 2001 From: Ilya Date: Mon, 3 Apr 2023 19:03:57 +0300 Subject: [PATCH 257/277] Fix missing metrics with multiple eval datasets (#22536) --- src/transformers/trainer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 615836e11604..de04f8b6741d 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -2243,12 +2243,14 @@ def _maybe_log_save_evaluate(self, tr_loss, model, trial, epoch, ignore_keys_for metrics = None if self.control.should_evaluate: if isinstance(self.eval_dataset, dict): + metrics = {} for eval_dataset_name, eval_dataset in self.eval_dataset.items(): - metrics = self.evaluate( + dataset_metrics = self.evaluate( eval_dataset=eval_dataset, ignore_keys=ignore_keys_for_eval, metric_key_prefix=f"eval_{eval_dataset_name}", ) + metrics.update(dataset_metrics) else: metrics = self.evaluate(ignore_keys=ignore_keys_for_eval) self._report_to_hp_search(trial, self.state.global_step, metrics) From 2cccab46ae8dab3c8a560a68ef346d631618b5b7 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 4 Apr 2023 00:04:24 +0800 Subject: [PATCH 258/277] [setup] drop deprecated `distutils` usage (#22531) * [setup] drop deprecated `distutils` usage * drop deprecated `distutils.util.strtobool` usage * fix import order * reformat docstring by `doc-builder` --- setup.py | 3 +-- src/transformers/testing_utils.py | 2 +- src/transformers/trainer.py | 2 +- src/transformers/utils/__init__.py | 1 + src/transformers/utils/generic.py | 15 +++++++++++++++ 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index ce609f240483..b24a57207310 100644 --- a/setup.py +++ b/setup.py @@ -76,10 +76,9 @@ import os import re import shutil -from distutils.core import Command from pathlib import Path -from setuptools import find_packages, setup +from setuptools import find_packages, setup, Command # Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466 diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 3fd334d71cb1..8d958a82509a 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -28,7 +28,6 @@ import time import unittest from collections.abc import Mapping -from distutils.util import strtobool from io import StringIO from pathlib import Path from typing import Iterator, List, Optional, Union @@ -93,6 +92,7 @@ is_torchdynamo_available, is_torchvision_available, is_vision_available, + strtobool, ) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index de04f8b6741d..ab79e94a68fe 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -29,7 +29,6 @@ import time import warnings from collections.abc import Mapping -from distutils.util import strtobool from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union @@ -152,6 +151,7 @@ is_torch_neuroncore_available, is_torch_tpu_available, logging, + strtobool, ) from .utils.generic import ContextManagers diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index b9b1771e6831..132c707f421b 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -47,6 +47,7 @@ is_torch_tensor, reshape, squeeze, + strtobool, tensor_size, to_numpy, to_py_obj, diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index 3c3ade198b35..65d92e195452 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -56,6 +56,21 @@ def __get__(self, obj, objtype=None): return cached +# vendored from distutils.util +def strtobool(val): + """Convert a string representation of truth to true (1) or false (0). + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values are 'n', 'no', 'f', 'false', 'off', and '0'. + Raises ValueError if 'val' is anything else. + """ + val = val.lower() + if val in {"y", "yes", "t", "true", "on", "1"}: + return 1 + if val in {"n", "no", "f", "false", "off", "0"}: + return 0 + raise ValueError(f"invalid truth value {val!r}") + + def is_tensor(x): """ Tests if `x` is a `torch.Tensor`, `tf.Tensor`, `jaxlib.xla_extension.DeviceArray` or `np.ndarray`. From a02907de5a4d68d16ee7f8f9c3e10af0c7882e06 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Mon, 3 Apr 2023 19:49:38 +0200 Subject: [PATCH 259/277] Generate: Enable easier TextStreamer customization (#22516) --- src/transformers/generation/streamers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/generation/streamers.py b/src/transformers/generation/streamers.py index 78d98666b3d1..06f7be9d63e0 100644 --- a/src/transformers/generation/streamers.py +++ b/src/transformers/generation/streamers.py @@ -88,7 +88,7 @@ def put(self, value): printable_text = text[self.print_len : text.rfind(" ") + 1] self.print_len += len(printable_text) - print(printable_text, flush=True, end="") + self.on_finalized_text(printable_text) def end(self): """Flushes any remaining cache and prints a newline to stdout.""" @@ -102,7 +102,11 @@ def end(self): printable_text = "" # Print a newline (and the remaining text, if any) - print(printable_text, flush=True) + self.on_finalized_text(printable_text, stream_end=True) + + def on_finalized_text(self, token: str, stream_end: bool = False): + """Prints the new text to stdout.""" + print(token, flush=True, end="" if not stream_end else None) class TextIteratorStreamer(BaseStreamer): From ecac59cc9131a0235e383b11ce94091827da46ad Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Tue, 4 Apr 2023 02:03:41 +0800 Subject: [PATCH 260/277] [setup] migrate setup script to `pyproject.toml` (#22539) * [setup] migrate setup script to `pyproject.toml` * [setup] cleanup configurations * remove unused imports --- pyproject.toml | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 17 ------------- setup.py | 31 +---------------------- 3 files changed, 69 insertions(+), 47 deletions(-) delete mode 100644 setup.cfg diff --git a/pyproject.toml b/pyproject.toml index dfb41c24d5d2..ef3be4606ea8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,68 @@ +# Package ###################################################################### + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "transformers" +description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +readme = "README.md" +requires-python = ">= 3.7" +authors = [ + { name = "The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)", email = "transformers@huggingface.co" }, +] +license = { text = "Apache 2.0 License" } +keywords = [ + "NLP", + "vision", + "speech", + "deep learning", + "transformer", + "pytorch", + "tensorflow", + "jax", + "BERT", + "GPT-2", + "Wav2Vec2", + "ViT", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dynamic = ["dependencies", "optional-dependencies", "version"] + +[project.scripts] +transformers-cli = "transformers.commands.transformers_cli:main" + +[project.urls] +Homepage = "https://huggingface.co" +Repository = "https://github.com/huggingface/transformers" +Documentation = "https://huggingface.co/docs/transformers" +"Bug Report" = "https://github.com/huggingface/transformers/issues" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +transformers = ["*.cu", "*.cpp", "*.cuh", "*.h", "*.pyx"] + +# Linter tools ################################################################# + [tool.black] line-length = 119 target-version = ['py37'] @@ -17,3 +82,6 @@ line-length = 119 [tool.ruff.isort] lines-after-imports = 2 known-first-party = ["transformers"] + +[tool.pytest] +doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 9a56ddc2fc65..000000000000 --- a/setup.cfg +++ /dev/null @@ -1,17 +0,0 @@ -[isort] -default_section = FIRSTPARTY -ensure_newline_before_comments = True -force_grid_wrap = 0 -include_trailing_comma = True -known_first_party = transformers -line_length = 119 -lines_after_imports = 2 -multi_line_output = 3 -use_parentheses = True - -[flake8] -ignore = E203, E501, E741, W503, W605 -max-line-length = 119 - -[tool:pytest] -doctest_optionflags=NUMBER NORMALIZE_WHITESPACE ELLIPSIS \ No newline at end of file diff --git a/setup.py b/setup.py index b24a57207310..0c42fb3f203d 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ import shutil from pathlib import Path -from setuptools import find_packages, setup, Command +from setuptools import setup, Command # Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466 @@ -425,36 +425,7 @@ def run(self): setup( name="transformers", version="4.28.0.dev0", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots) - author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)", - author_email="transformers@huggingface.co", - description="State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow", - long_description=open("README.md", "r", encoding="utf-8").read(), - long_description_content_type="text/markdown", - keywords="NLP vision speech deep learning transformer pytorch tensorflow jax BERT GPT-2 Wav2Vec2 ViT", - license="Apache 2.0 License", - url="https://github.com/huggingface/transformers", - package_dir={"": "src"}, - packages=find_packages("src"), - include_package_data=True, - package_data={"transformers": ["*.cu", "*.cpp", "*.cuh", "*.h", "*.pyx"]}, - zip_safe=False, extras_require=extras, - entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]}, - python_requires=">=3.7.0", install_requires=install_requires, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - ], cmdclass={"deps_table_update": DepsTableUpdateCommand}, ) From d5372fdb21c35e043c3050ac0f0b8b0719b9d17a Mon Sep 17 00:00:00 2001 From: Sylvain Gugger Date: Mon, 3 Apr 2023 14:07:40 -0400 Subject: [PATCH 261/277] Skip failing test --- tests/models/pix2struct/test_image_processing_pix2struct.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models/pix2struct/test_image_processing_pix2struct.py b/tests/models/pix2struct/test_image_processing_pix2struct.py index e13e38b71c02..667ae477403c 100644 --- a/tests/models/pix2struct/test_image_processing_pix2struct.py +++ b/tests/models/pix2struct/test_image_processing_pix2struct.py @@ -87,6 +87,7 @@ def test_image_processor_properties(self): self.assertTrue(hasattr(image_processor, "do_normalize")) self.assertTrue(hasattr(image_processor, "do_convert_rgb")) + @unittest.skip("fix me Younes.") def test_expected_patches(self): dummy_image = self.image_processor_tester.prepare_dummy_image() From 14f22d10f6eaab0fe2eaf2d13fecdccccaaa65cf Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 3 Apr 2023 21:26:35 +0200 Subject: [PATCH 262/277] Update test_image_processing_pix2struct.py (#22543) --- tests/models/pix2struct/test_image_processing_pix2struct.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/models/pix2struct/test_image_processing_pix2struct.py b/tests/models/pix2struct/test_image_processing_pix2struct.py index 667ae477403c..fd805da696b9 100644 --- a/tests/models/pix2struct/test_image_processing_pix2struct.py +++ b/tests/models/pix2struct/test_image_processing_pix2struct.py @@ -65,7 +65,7 @@ def prepare_image_processor_dict(self): return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb} def prepare_dummy_image(self): - img_url = "https://www.ilankelman.org/stopsigns/australia.jpg" + img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg" raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB") return raw_image @@ -87,7 +87,6 @@ def test_image_processor_properties(self): self.assertTrue(hasattr(image_processor, "do_normalize")) self.assertTrue(hasattr(image_processor, "do_convert_rgb")) - @unittest.skip("fix me Younes.") def test_expected_patches(self): dummy_image = self.image_processor_tester.prepare_dummy_image() From 889607f6a157d0088c3a2072e3045f5698f10fb9 Mon Sep 17 00:00:00 2001 From: Pavel T Date: Mon, 3 Apr 2023 21:05:31 -0400 Subject: [PATCH 263/277] Fix OPTForQuestionAnswering doc string (#22481) * Fix OPTForQuestionAnswering doc string for more adequate model answer decoding * black style fix * doc-builder style --- src/transformers/models/opt/modeling_opt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 6ecedc2173e7..f9eac7d915c7 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -1195,10 +1195,14 @@ def forward( >>> answer_start_index = outputs.start_logits.argmax() >>> answer_end_index = outputs.end_logits.argmax() - >>> predict_answer_tokens = inputs.input_ids[0, answer_start_index : answer_end_index + 1] + >>> answer_offset = len(tokenizer(question)[0]) + + >>> predict_answer_tokens = inputs.input_ids[ + ... 0, answer_offset + answer_start_index : answer_offset + answer_end_index + 1 + ... ] >>> predicted = tokenizer.decode(predict_answer_tokens) >>> predicted - ' Henson?' + ' a nice puppet' ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict From 995ad7b48001d47ff481ffd0a14e7ded3a5035ce Mon Sep 17 00:00:00 2001 From: Joao Gante Date: Tue, 4 Apr 2023 09:03:13 +0100 Subject: [PATCH 264/277] Generate: Add text streamer decoding options (#22544) --- src/transformers/generation/streamers.py | 94 ++++++++++-------------- tests/generation/test_streamers.py | 39 ++++++++++ 2 files changed, 77 insertions(+), 56 deletions(-) diff --git a/src/transformers/generation/streamers.py b/src/transformers/generation/streamers.py index 06f7be9d63e0..719c067d21c0 100644 --- a/src/transformers/generation/streamers.py +++ b/src/transformers/generation/streamers.py @@ -42,6 +42,10 @@ class TextStreamer(BaseStreamer): Parameters: tokenizer (`AutoTokenizer`): The tokenized used to decode the tokens. + skip_prompt (`bool`, *optional*, defaults to `False`): + Whether to skip the prompt to `.generate()` or not. Useful e.g. for chatbots. + decode_kwargs (`dict`, *optional*): + Additional keyword arguments to pass to the tokenizer's `decode` method. Examples: @@ -59,10 +63,15 @@ class TextStreamer(BaseStreamer): ``` """ - def __init__(self, tokenizer: "AutoTokenizer"): + def __init__(self, tokenizer: "AutoTokenizer", skip_prompt: bool = False, **decode_kwargs): self.tokenizer = tokenizer + self.skip_prompt = skip_prompt + self.decode_kwargs = decode_kwargs + + # variables used in the streaming process self.token_cache = [] self.print_len = 0 + self.next_tokens_are_prompt = True def put(self, value): """ @@ -73,11 +82,15 @@ def put(self, value): elif len(value.shape) > 1: value = value[0] + if self.skip_prompt and self.next_tokens_are_prompt: + self.next_tokens_are_prompt = False + return + # Add the new token to the cache and decodes the entire thing. self.token_cache.extend(value.tolist()) - text = self.tokenizer.decode(self.token_cache) + text = self.tokenizer.decode(self.token_cache, **self.decode_kwargs) - # After symbol for a new line, we flush the cache. + # After the symbol for a new line, we flush the cache. if text.endswith("\n"): printable_text = text[self.print_len :] self.token_cache = [] @@ -94,30 +107,34 @@ def end(self): """Flushes any remaining cache and prints a newline to stdout.""" # Flush the cache, if it exists if len(self.token_cache) > 0: - text = self.tokenizer.decode(self.token_cache) + text = self.tokenizer.decode(self.token_cache, **self.decode_kwargs) printable_text = text[self.print_len :] self.token_cache = [] self.print_len = 0 else: printable_text = "" - # Print a newline (and the remaining text, if any) + self.next_tokens_are_prompt = True self.on_finalized_text(printable_text, stream_end=True) - def on_finalized_text(self, token: str, stream_end: bool = False): - """Prints the new text to stdout.""" - print(token, flush=True, end="" if not stream_end else None) + def on_finalized_text(self, text: str, stream_end: bool = False): + """Prints the new text to stdout. If the stream is ending, also prints a newline.""" + print(text, flush=True, end="" if not stream_end else None) -class TextIteratorStreamer(BaseStreamer): +class TextIteratorStreamer(TextStreamer): """ Streamer that stores print-ready text in a queue, to be used by a downstream application as an iterator. This is - useful for applications that want to use the generated text in a non-blocking way (e.g. in an interactive Gradio - demo). + useful for applications that benefit from acessing the generated text in a non-blocking way (e.g. in an interactive + Gradio demo). Parameters: tokenizer (`AutoTokenizer`): The tokenized used to decode the tokens. + skip_prompt (`bool`, *optional*, defaults to `False`): + Whether to skip the prompt to `.generate()` or not. Useful e.g. for chatbots. + decode_kwargs (`dict`, *optional*): + Additional keyword arguments to pass to the tokenizer's `decode` method. Examples: @@ -142,58 +159,23 @@ class TextIteratorStreamer(BaseStreamer): ``` """ - def __init__(self, tokenizer: "AutoTokenizer"): - self.tokenizer = tokenizer - self.token_cache = [] - self.print_len = 0 - self.queue = Queue() + def __init__(self, tokenizer: "AutoTokenizer", skip_prompt: bool = False, **decode_kwargs): + super().__init__(tokenizer, skip_prompt, **decode_kwargs) + self.text_queue = Queue() self.stop_signal = None + def on_finalized_text(self, text: str, stream_end: bool = False): + """Put the new text in the queue. If the stream is ending, also put a stop signal in the queue.""" + self.text_queue.put(text) + if stream_end: + self.text_queue.put(self.stop_signal) + def __iter__(self): return self def __next__(self): - value = self.queue.get() + value = self.text_queue.get() if value == self.stop_signal: raise StopIteration() else: return value - - def put(self, value): - """ - Recives tokens, decodes them, and pushes text to the queue as soon as it form entire words. - """ - if len(value.shape) > 1 and value.shape[0] > 1: - raise ValueError("TextStreamer only supports batch size 1") - elif len(value.shape) > 1: - value = value[0] - - # Add the new token to the cache and decodes the entire thing. - self.token_cache.extend(value.tolist()) - text = self.tokenizer.decode(self.token_cache) - - # After symbol for a new line, we flush the cache. - if text.endswith("\n"): - printable_text = text[self.print_len :] - self.token_cache = [] - self.print_len = 0 - # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, - # which may change with the subsequent token -- there are probably smarter ways to do this!) - else: - printable_text = text[self.print_len : text.rfind(" ") + 1] - self.print_len += len(printable_text) - self.queue.put(printable_text) - - def end(self): - """Flushes any remaining cache and puts the stop signal in the queue.""" - # Flush the cache, if it exists - if len(self.token_cache) > 0: - text = self.tokenizer.decode(self.token_cache) - printable_text = text[self.print_len :] - self.token_cache = [] - self.print_len = 0 - else: - printable_text = "" - - self.queue.put(printable_text) - self.queue.put(self.stop_signal) # Put the stop signal diff --git a/tests/generation/test_streamers.py b/tests/generation/test_streamers.py index bf465ab31c6b..7214e56cd3c3 100644 --- a/tests/generation/test_streamers.py +++ b/tests/generation/test_streamers.py @@ -23,6 +23,8 @@ if is_torch_available(): + import torch + from transformers import AutoModelForCausalLM @@ -63,3 +65,40 @@ def test_iterator_streamer_matches_non_streaming(self): streamer_text += new_text self.assertEqual(streamer_text, greedy_text) + + def test_text_streamer_skip_prompt(self): + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2") + model = AutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-gpt2").to(torch_device) + model.config.eos_token_id = -1 + + input_ids = ids_tensor((1, 5), vocab_size=model.config.vocab_size).to(torch_device) + greedy_ids = model.generate(input_ids, max_new_tokens=10, do_sample=False) + new_greedy_ids = greedy_ids[:, input_ids.shape[1] :] + new_greedy_text = tokenizer.decode(new_greedy_ids[0]) + + with CaptureStdout() as cs: + streamer = TextStreamer(tokenizer, skip_prompt=True) + model.generate(input_ids, max_new_tokens=10, do_sample=False, streamer=streamer) + # The greedy text should be printed to stdout, except for the final "\n" in the streamer + streamer_text = cs.out[:-1] + + self.assertEqual(streamer_text, new_greedy_text) + + def test_text_streamer_decode_kwargs(self): + # Tests that we can pass `decode_kwargs` to the streamer to control how the tokens are decoded. Must be tested + # with actual models -- the dummy models' tokenizers are not aligned with their models, and + # `skip_special_tokens=True` has no effect on them + tokenizer = AutoTokenizer.from_pretrained("distilgpt2") + model = AutoModelForCausalLM.from_pretrained("distilgpt2").to(torch_device) + model.config.eos_token_id = -1 + + input_ids = torch.ones((1, 5), device=torch_device).long() * model.config.bos_token_id + with CaptureStdout() as cs: + streamer = TextStreamer(tokenizer, skip_special_tokens=True) + model.generate(input_ids, max_new_tokens=1, do_sample=False, streamer=streamer) + + # The prompt contains a special token, so the streamer should not print it. As such, the output text, when + # re-tokenized, must only contain one token + streamer_text = cs.out[:-1] # Remove the final "\n" + streamer_text_tokenized = tokenizer(streamer_text, return_tensors="pt") + self.assertEqual(streamer_text_tokenized.input_ids.shape, (1, 1)) From b8edf8794cb77c1d846ba7c66568e53e1d5d851f Mon Sep 17 00:00:00 2001 From: TheWall9 Date: Tue, 4 Apr 2023 18:50:33 +0800 Subject: [PATCH 265/277] [Roformer] Fixing a bug in RoFormerEncoder where it was ignoring the length of past_key_values when generating as a decoder (#22416) * fix RoFormerEncoder postion embedding when generate as decoder * make fixup * add test case for check generate with past key values * remove duplicating code --- .../models/roformer/modeling_roformer.py | 12 +++++----- .../models/roformer/test_modeling_roformer.py | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/roformer/modeling_roformer.py b/src/transformers/models/roformer/modeling_roformer.py index e3e1d4b6be06..d6752f98d725 100644 --- a/src/transformers/models/roformer/modeling_roformer.py +++ b/src/transformers/models/roformer/modeling_roformer.py @@ -259,11 +259,6 @@ def forward( key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) attention_mask = encoder_attention_mask - elif past_key_value is not None: - key_layer = self.transpose_for_scores(self.key(hidden_states)) - value_layer = self.transpose_for_scores(self.value(hidden_states)) - key_layer = torch.cat([past_key_value[0], key_layer], dim=2) - value_layer = torch.cat([past_key_value[1], value_layer], dim=2) else: key_layer = self.transpose_for_scores(self.key(hidden_states)) value_layer = self.transpose_for_scores(self.value(hidden_states)) @@ -276,6 +271,9 @@ def forward( query_layer, key_layer = self.apply_rotary_position_embeddings( sinusoidal_pos, query_layer, key_layer ) + if past_key_value is not None: + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) if self.is_decoder: # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. # Further calls to cross_attention layer can then reuse all cross-attention @@ -566,8 +564,10 @@ def forward( all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 + # [sequence_length, embed_size_per_head] -> [batch_size, num_heads, sequence_length, embed_size_per_head] - sinusoidal_pos = self.embed_positions(hidden_states.shape[:-1])[None, None, :, :] + sinusoidal_pos = self.embed_positions(hidden_states.shape[:-1], past_key_values_length)[None, None, :, :] next_decoder_cache = () if use_cache else None for i, layer_module in enumerate(self.layer): diff --git a/tests/models/roformer/test_modeling_roformer.py b/tests/models/roformer/test_modeling_roformer.py index 9b91c66cb2fa..357e126a047a 100644 --- a/tests/models/roformer/test_modeling_roformer.py +++ b/tests/models/roformer/test_modeling_roformer.py @@ -220,6 +220,25 @@ def create_and_check_for_causal_lm( result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + def create_and_check_for_generate_causal_lm( + self, + config, + input_ids, + token_type_ids, + input_mask, + sequence_labels, + token_labels, + choice_labels, + ): + model = RoFormerForCausalLM(config=config).to(torch_device).eval() + torch.manual_seed(0) + output_without_past_cache = model.generate( + input_ids[:1], num_beams=2, max_length=15, do_sample=True, use_cache=False + ) + torch.manual_seed(0) + output_with_past_cache = model.generate(input_ids[:1], num_beams=2, max_length=15, do_sample=True) + self.parent.assertTrue(torch.all(output_with_past_cache == output_without_past_cache)) + def create_and_check_for_masked_lm( self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels ): @@ -405,6 +424,10 @@ def test_for_masked_lm(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_for_masked_lm(*config_and_inputs) + def test_for_generate_causal_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_generate_causal_lm(*config_and_inputs) + def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs) From 54e01822d525714b8bff30aed78373fdd257be4f Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Tue, 4 Apr 2023 14:53:06 +0200 Subject: [PATCH 266/277] =?UTF-8?q?=F0=9F=9A=A8=F0=9F=9A=A8=F0=9F=9A=A8=20?= =?UTF-8?q?`[NLLB=20Tokenizer]`=20Fix=20the=20prefix=20tokens=20?= =?UTF-8?q?=F0=9F=9A=A8=F0=9F=9A=A8=F0=9F=9A=A8=20(#22313)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix the prefix tokens * update fast and test values * add legacy behaviour Co-authored-by: sgugger * update disclaimer, linkissue PR and behaviral changes * Apply suggestions from code review Co-authored-by: Lysandre Debut * styling * make a quote * quote this time --------- Co-authored-by: sgugger Co-authored-by: Lysandre Debut --- docs/source/en/model_doc/nllb.mdx | 41 ++++++++++++++++++- .../models/nllb/tokenization_nllb.py | 30 ++++++++++---- .../models/nllb/tokenization_nllb_fast.py | 31 ++++++++++---- tests/models/nllb/test_tokenization_nllb.py | 32 +++++++++++---- 4 files changed, 111 insertions(+), 23 deletions(-) diff --git a/docs/source/en/model_doc/nllb.mdx b/docs/source/en/model_doc/nllb.mdx index 9ad286164ba0..60819da998d0 100644 --- a/docs/source/en/model_doc/nllb.mdx +++ b/docs/source/en/model_doc/nllb.mdx @@ -12,8 +12,45 @@ specific language governing permissions and limitations under the License. # NLLB -**DISCLAIMER:** If you see something strange, file a [Github Issue](https://github.com/huggingface/transformers/issues/new?assignees=&labels=bug&template=bug-report.yml) and assign -@LysandreJik +**DISCLAIMER:** The default behaviour for the tokenizer has recently been fixed (and thus changed)! + +The previous version adds `[self.eos_token_id, self.cur_lang_code]` at the end of the token sequence for both target and source tokenization. This is wrong as the NLLB paper mentions (page 48, 6.1.1. Model Architecture) : + +*Note that we prefix the source sequence with the source language, as opposed to the target +language as previously done in several works (Arivazhagan et al., 2019; Johnson et al., +2017). This is primarily because we prioritize optimizing zero-shot performance of our +model on any pair of 200 languages at a minor cost to supervised performance.* + +Previous behaviour: + +```python +>>> from transformers import NllbTokenizer + +>>> tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-600M") +>>> tokenizer("How was your day?").input_ids +[13374, 1398, 4260, 4039, 248130, 2, 256047] + +>>> # 2: '' +>>> # 256047 : 'eng_Latn' +``` +New behaviour + +```python +>>> from transformers import NllbTokenizer + +>>> tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-600M") +>>> tokenizer("How was your day?").input_ids +[256047, 13374, 1398, 4260, 4039, 248130, 2] + ``` + +Enabling the old behaviour can be done as follows: +```python +>>> from transformers import NllbTokenizer + +>>> tokenizer = NllbTokenizer.from_pretrained("facebook/nllb-200-distilled-600M", legacy_behaviour=True) +``` + +For more details, feel free to check the linked [PR](https://github.com/huggingface/transformers/pull/22313) and [Issue](https://github.com/huggingface/transformers/issues/19943). ## Overview of NLLB diff --git a/src/transformers/models/nllb/tokenization_nllb.py b/src/transformers/models/nllb/tokenization_nllb.py index ac2aa2380b1b..58a02a7af75f 100644 --- a/src/transformers/models/nllb/tokenization_nllb.py +++ b/src/transformers/models/nllb/tokenization_nllb.py @@ -140,12 +140,14 @@ def __init__( tgt_lang=None, sp_model_kwargs: Optional[Dict[str, Any]] = None, additional_special_tokens=None, + legacy_behaviour=False, **kwargs, ): # Mask token behave like a normal word, i.e. include the space before it mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + self.legacy_behaviour = legacy_behaviour super().__init__( bos_token=bos_token, @@ -160,13 +162,13 @@ def __init__( tgt_lang=tgt_lang, additional_special_tokens=additional_special_tokens, sp_model_kwargs=self.sp_model_kwargs, + legacy_behaviour=legacy_behaviour, **kwargs, ) self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) self.sp_model.Load(str(vocab_file)) self.vocab_file = vocab_file - # Original fairseq vocab and spm vocab must be "aligned": # Vocab | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 # -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ---- @@ -388,13 +390,27 @@ def _switch_to_target_mode(self): return self.set_tgt_lang_special_tokens(self.tgt_lang) def set_src_lang_special_tokens(self, src_lang) -> None: - """Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].""" + """Reset the special tokens to the source lang setting. + - In legacy mode: No prefix and suffix=[eos, src_lang_code]. + - In default mode: Prefix=[src_lang_code], suffix = [eos] + """ self.cur_lang_code = self.lang_code_to_id[src_lang] - self.prefix_tokens = [] - self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + if self.legacy_behaviour: + self.prefix_tokens = [] + self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + else: + self.prefix_tokens = [self.cur_lang_code] + self.suffix_tokens = [self.eos_token_id] def set_tgt_lang_special_tokens(self, lang: str) -> None: - """Reset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].""" + """Reset the special tokens to the target lang setting. + - In legacy mode: No prefix and suffix=[eos, tgt_lang_code]. + - In default mode: Prefix=[tgt_lang_code], suffix = [eos] + """ self.cur_lang_code = self.lang_code_to_id[lang] - self.prefix_tokens = [] - self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + if self.legacy_behaviour: + self.prefix_tokens = [] + self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + else: + self.prefix_tokens = [self.cur_lang_code] + self.suffix_tokens = [self.eos_token_id] diff --git a/src/transformers/models/nllb/tokenization_nllb_fast.py b/src/transformers/models/nllb/tokenization_nllb_fast.py index 7c6979e295b9..9eeaf5d87eb1 100644 --- a/src/transformers/models/nllb/tokenization_nllb_fast.py +++ b/src/transformers/models/nllb/tokenization_nllb_fast.py @@ -151,11 +151,12 @@ def __init__( src_lang=None, tgt_lang=None, additional_special_tokens=None, + legacy_behaviour=False, **kwargs, ): # Mask token behave like a normal word, i.e. include the space before it mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token - + self.legacy_behaviour = legacy_behaviour super().__init__( vocab_file=vocab_file, tokenizer_file=tokenizer_file, @@ -169,6 +170,7 @@ def __init__( src_lang=src_lang, tgt_lang=tgt_lang, additional_special_tokens=additional_special_tokens, + legacy_behaviour=legacy_behaviour, **kwargs, ) @@ -287,10 +289,18 @@ def _switch_to_target_mode(self): return self.set_tgt_lang_special_tokens(self.tgt_lang) def set_src_lang_special_tokens(self, src_lang) -> None: - """Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].""" + """Reset the special tokens to the source lang setting. + - In legacy mode: No prefix and suffix=[eos, src_lang_code]. + - In default mode: Prefix=[src_lang_code], suffix = [eos] + """ self.cur_lang_code = self.convert_tokens_to_ids(src_lang) - self.prefix_tokens = [] - self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + + if self.legacy_behaviour: + self.prefix_tokens = [] + self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + else: + self.prefix_tokens = [self.cur_lang_code] + self.suffix_tokens = [self.eos_token_id] prefix_tokens_str = self.convert_ids_to_tokens(self.prefix_tokens) suffix_tokens_str = self.convert_ids_to_tokens(self.suffix_tokens) @@ -302,10 +312,17 @@ def set_src_lang_special_tokens(self, src_lang) -> None: ) def set_tgt_lang_special_tokens(self, lang: str) -> None: - """Reset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].""" + """Reset the special tokens to the target lang setting. + - In legacy mode: No prefix and suffix=[eos, tgt_lang_code]. + - In default mode: Prefix=[tgt_lang_code], suffix = [eos] + """ self.cur_lang_code = self.convert_tokens_to_ids(lang) - self.prefix_tokens = [] - self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + if self.legacy_behaviour: + self.prefix_tokens = [] + self.suffix_tokens = [self.eos_token_id, self.cur_lang_code] + else: + self.prefix_tokens = [self.cur_lang_code] + self.suffix_tokens = [self.eos_token_id] prefix_tokens_str = self.convert_ids_to_tokens(self.prefix_tokens) suffix_tokens_str = self.convert_ids_to_tokens(self.suffix_tokens) diff --git a/tests/models/nllb/test_tokenization_nllb.py b/tests/models/nllb/test_tokenization_nllb.py index 01d0420180c8..f0e74d2f82de 100644 --- a/tests/models/nllb/test_tokenization_nllb.py +++ b/tests/models/nllb/test_tokenization_nllb.py @@ -305,6 +305,7 @@ class NllbDistilledIntegrationTest(unittest.TestCase): " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.", ] expected_src_tokens = [ + 256047, 16297, 134408, 8165, @@ -319,7 +320,6 @@ class NllbDistilledIntegrationTest(unittest.TestCase): 108, 49486, 2, - 256047, ] @classmethod @@ -355,8 +355,8 @@ def test_enro_tokenizer_truncation(self): assert isinstance(src_text[0], str) desired_max_length = 10 ids = self.tokenizer(src_text, max_length=desired_max_length, truncation=True).input_ids[0] - self.assertEqual(ids[-2], 2) - self.assertEqual(ids[-1], EN_CODE) + self.assertEqual(ids[-1], 2) + self.assertEqual(ids[0], EN_CODE) self.assertEqual(len(ids), desired_max_length) def test_mask_token(self): @@ -389,10 +389,10 @@ def test_enro_tokenizer_prepare_batch(self): self.assertEqual((2, 15), batch.attention_mask.shape) result = batch.input_ids.tolist()[0] self.assertListEqual(self.expected_src_tokens, result) - self.assertEqual(2, batch.decoder_input_ids[0, -1]) # EOS + self.assertEqual(RO_CODE, batch.decoder_input_ids[0, 0]) # EOS # Test that special tokens are reset - self.assertEqual(self.tokenizer.prefix_tokens, []) - self.assertEqual(self.tokenizer.suffix_tokens, [self.tokenizer.eos_token_id, EN_CODE]) + self.assertEqual(self.tokenizer.prefix_tokens, [EN_CODE]) + self.assertEqual(self.tokenizer.suffix_tokens, [self.tokenizer.eos_token_id]) def test_seq2seq_max_length(self): batch = self.tokenizer(self.src_text, padding=True, truncation=True, max_length=3, return_tensors="pt") @@ -419,9 +419,27 @@ def test_tokenizer_translation(self): nested_simplify(inputs), { # A, test, EOS, en_XX - "input_ids": [[70, 7356, 2, 256047]], + "input_ids": [[256047, 70, 7356, 2]], "attention_mask": [[1, 1, 1, 1]], # ar_AR "forced_bos_token_id": 256057, }, ) + + @require_torch + def test_legacy_behaviour(self): + self.tokenizer.legacy_behaviour = True + inputs = self.tokenizer( + "UN Chief says there is no military solution in Syria", src_lang="eng_Latn", tgt_lang="fra_Latn" + ) + self.assertEqual( + inputs.input_ids, [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047] + ) + + self.tokenizer.legacy_behaviour = False + inputs = self.tokenizer( + "UN Chief says there is no military solution in Syria", src_lang="eng_Latn", tgt_lang="fra_Latn" + ) + self.assertEqual( + inputs.input_ids, [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2] + ) From 53e5389679454ffe61d164fc6f1a4fb5ce24d957 Mon Sep 17 00:00:00 2001 From: Viktor Scherbakov Date: Tue, 4 Apr 2023 16:05:04 +0300 Subject: [PATCH 267/277] Implemented safetensors checkpoints save/load for Trainer (#22498) * implemented safetensors save/load * remove duplicated file * added tests * more tests * style fix * fix tf tests * change to list comprehension Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * review fixes + safe load for sharded checkpoint * style fix * remove rogue import * remove partial to avoid undefined exception * use naming alias instead of safetensors.torch * fix safe sharding in tests * grammar Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * update docs Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * update docs Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * minor corrections * style --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/modeling_utils.py | 39 +++++++-- src/transformers/trainer.py | 74 +++++++++++----- src/transformers/training_args.py | 21 +++++ tests/trainer/test_trainer.py | 133 ++++++++++++++++++++++++++--- 4 files changed, 231 insertions(+), 36 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 9a6c29c27bdf..27faa252788d 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -336,7 +336,7 @@ def shard_checkpoint( return shards, index -def load_sharded_checkpoint(model, folder, strict=True): +def load_sharded_checkpoint(model, folder, strict=True, prefer_safe=True): """ This is the same as [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict) @@ -350,6 +350,9 @@ def load_sharded_checkpoint(model, folder, strict=True): folder (`str` or `os.PathLike`): A path to a folder containing the sharded checkpoint. strict (`bool`, *optional`, defaults to `True`): Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint. + prefer_safe (`bool`, *optional*, defaults to `False`) + If both safetensors and PyTorch save files are present in checkpoint and `prefer_safe` is True, the + safetensors files will be loaded. Otherwise, PyTorch files are always loaded when possible. Returns: `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields @@ -358,10 +361,32 @@ def load_sharded_checkpoint(model, folder, strict=True): """ # Load the index index_file = os.path.join(folder, WEIGHTS_INDEX_NAME) - if not os.path.isfile(index_file): - raise ValueError(f"Can't find a checkpoint index ({WEIGHTS_INDEX_NAME}) in {folder}.") + safe_index_file = os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME) - with open(index_file, "r", encoding="utf-8") as f: + index_present = os.path.isfile(index_file) + safe_index_present = os.path.isfile(safe_index_file) + + if not index_present and not (safe_index_present and is_safetensors_available()): + filenames = ( + (WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_INDEX_NAME) if is_safetensors_available() else (WEIGHTS_INDEX_NAME,) + ) + raise ValueError(f"Can't find a checkpoint index ({' or '.join(filenames)}) in {folder}.") + + load_safe = False + if safe_index_present: + if prefer_safe: + if is_safetensors_available(): + load_safe = True # load safe due to preference + else: + logger.warning( + f"Cannot load sharded checkpoint at {folder} safely since safetensors is not installed!" + ) + elif not index_present: + load_safe = True # load safe since we have no other choice + + load_index = safe_index_file if load_safe else index_file + + with open(load_index, "r", encoding="utf-8") as f: index = json.load(f) shard_files = list(set(index["weight_map"].values())) @@ -381,11 +406,13 @@ def load_sharded_checkpoint(model, folder, strict=True): error_message += f"\nMissing key(s): {str_unexpected_keys}." raise RuntimeError(error_message) + loader = safe_load_file if load_safe else partial(torch.load, map_location="cpu") + for shard_file in shard_files: - state_dict = torch.load(os.path.join(folder, shard_file), map_location="cpu") + state_dict = loader(os.path.join(folder, shard_file)) model.load_state_dict(state_dict, strict=False) - # Make sure memory is fred before we load the next state dict. + # Make sure memory is freed before we load the next state dict. del state_dict gc.collect() diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index ab79e94a68fe..2eb081af7c6c 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -135,6 +135,8 @@ from .training_args import OptimizerNames, ParallelMode, TrainingArguments from .utils import ( CONFIG_NAME, + SAFE_WEIGHTS_INDEX_NAME, + SAFE_WEIGHTS_NAME, WEIGHTS_INDEX_NAME, WEIGHTS_NAME, can_return_loss, @@ -145,6 +147,7 @@ is_datasets_available, is_in_notebook, is_ipex_available, + is_safetensors_available, is_sagemaker_dp_enabled, is_sagemaker_mp_enabled, is_torch_compile_available, @@ -198,6 +201,10 @@ IS_SAGEMAKER_MP_POST_1_10 = False +if is_safetensors_available(): + import safetensors.torch + + skip_first_batches = None if is_accelerate_available(): from accelerate import __version__ as accelerate_version @@ -2091,15 +2098,22 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None): if model is None: model = self.model - if not os.path.isfile(os.path.join(resume_from_checkpoint, WEIGHTS_NAME)) and not os.path.isfile( - os.path.join(resume_from_checkpoint, WEIGHTS_INDEX_NAME) + config_file = os.path.join(resume_from_checkpoint, CONFIG_NAME) + + weights_file = os.path.join(resume_from_checkpoint, WEIGHTS_NAME) + weights_index_file = os.path.join(resume_from_checkpoint, WEIGHTS_INDEX_NAME) + safe_weights_file = os.path.join(resume_from_checkpoint, SAFE_WEIGHTS_NAME) + safe_weights_index_file = os.path.join(resume_from_checkpoint, SAFE_WEIGHTS_INDEX_NAME) + + if not any( + [os.path.isfile(f) for f in [weights_file, safe_weights_file, weights_index_file, safe_weights_index_file]] ): raise ValueError(f"Can't find a valid checkpoint at {resume_from_checkpoint}") logger.info(f"Loading model from {resume_from_checkpoint}.") - if os.path.isfile(os.path.join(resume_from_checkpoint, CONFIG_NAME)): - config = PretrainedConfig.from_json_file(os.path.join(resume_from_checkpoint, CONFIG_NAME)) + if os.path.isfile(config_file): + config = PretrainedConfig.from_json_file(config_file) checkpoint_version = config.transformers_version if checkpoint_version is not None and checkpoint_version != __version__: logger.warning( @@ -2108,7 +2122,7 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None): "yield to errors or unwanted behaviors." ) - if os.path.isfile(os.path.join(resume_from_checkpoint, WEIGHTS_NAME)): + if os.path.isfile(weights_file) or os.path.isfile(safe_weights_file): # If the model is on the GPU, it still works! if is_sagemaker_mp_enabled(): if os.path.isfile(os.path.join(resume_from_checkpoint, "user_content.pt")): @@ -2124,7 +2138,7 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None): logger.warning( "Enabling FP16 and loading from smp < 1.10 checkpoint together is not suppported." ) - state_dict = torch.load(os.path.join(resume_from_checkpoint, WEIGHTS_NAME), map_location="cpu") + state_dict = torch.load(weights_file, map_location="cpu") # Required for smp to not auto-translate state_dict from hf to smp (is already smp). state_dict["_smp_is_partial"] = False load_result = model.load_state_dict(state_dict, strict=True) @@ -2132,7 +2146,11 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None): del state_dict else: # We load the model state dict on the CPU to avoid an OOM error. - state_dict = torch.load(os.path.join(resume_from_checkpoint, WEIGHTS_NAME), map_location="cpu") + if self.args.save_safetensors and os.path.isfile(safe_weights_file): + state_dict = safetensors.torch.load_file(safe_weights_file, device="cpu") + else: + state_dict = torch.load(weights_file, map_location="cpu") + # workaround for FSDP bug https://github.com/pytorch/pytorch/issues/82963 # which takes *args instead of **kwargs load_result = model.load_state_dict(state_dict, False) @@ -2141,15 +2159,18 @@ def _load_from_checkpoint(self, resume_from_checkpoint, model=None): self._issue_warnings_after_load(load_result) else: # We load the sharded checkpoint - load_result = load_sharded_checkpoint(model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled()) + load_result = load_sharded_checkpoint( + model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled(), prefer_safe=self.args.save_safetensors + ) if not is_sagemaker_mp_enabled(): self._issue_warnings_after_load(load_result) def _load_best_model(self): logger.info(f"Loading best model from {self.state.best_model_checkpoint} (score: {self.state.best_metric}).") best_model_path = os.path.join(self.state.best_model_checkpoint, WEIGHTS_NAME) + best_safe_model_path = os.path.join(self.state.best_model_checkpoint, SAFE_WEIGHTS_NAME) model = self.model_wrapped if is_sagemaker_mp_enabled() else self.model - if os.path.exists(best_model_path): + if os.path.exists(best_model_path) or os.path.exists(best_safe_model_path): if self.deepspeed: if self.model_wrapped is not None: # this removes the pre-hooks from the previous engine @@ -2181,12 +2202,20 @@ def _load_best_model(self): else: # If the 'user_content.pt' file does NOT exist, load with the old smp api. # Checkpoint must have been saved with the old smp api. - state_dict = torch.load(best_model_path, map_location="cpu") + if self.args.save_safetensors and os.path.isfile(best_safe_model_path): + state_dict = safetensors.torch.load_file(best_safe_model_path, device="cpu") + else: + state_dict = torch.load(best_model_path, map_location="cpu") + state_dict["_smp_is_partial"] = False load_result = model.load_state_dict(state_dict, strict=True) else: # We load the model state dict on the CPU to avoid an OOM error. - state_dict = torch.load(best_model_path, map_location="cpu") + if self.args.save_safetensors and os.path.isfile(best_safe_model_path): + state_dict = safetensors.torch.load_file(best_safe_model_path, device="cpu") + else: + state_dict = torch.load(best_model_path, map_location="cpu") + # If the model is on the GPU, it still works! # workaround for FSDP bug https://github.com/pytorch/pytorch/issues/82963 # which takes *args instead of **kwargs @@ -2837,17 +2866,24 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): # Save a trained model and configuration using `save_pretrained()`. # They can then be reloaded using `from_pretrained()` if not isinstance(self.model, PreTrainedModel): + if state_dict is None: + state_dict = self.model.state_dict() + if isinstance(unwrap_model(self.model), PreTrainedModel): - if state_dict is None: - state_dict = self.model.state_dict() - unwrap_model(self.model).save_pretrained(output_dir, state_dict=state_dict) + unwrap_model(self.model).save_pretrained( + output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors + ) else: logger.info("Trainer.model is not a `PreTrainedModel`, only saving its state dict.") - if state_dict is None: - state_dict = self.model.state_dict() - torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME)) + if self.args.save_safetensors: + safetensors.torch.save_file(state_dict, os.path.join(output_dir, SAFE_WEIGHTS_NAME)) + else: + torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME)) else: - self.model.save_pretrained(output_dir, state_dict=state_dict) + self.model.save_pretrained( + output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors + ) + if self.tokenizer is not None: self.tokenizer.save_pretrained(output_dir) @@ -3546,7 +3582,7 @@ def _push_from_checkpoint(self, checkpoint_folder): output_dir = self.args.output_dir # To avoid a new synchronization of all model weights, we just copy the file from the checkpoint folder - modeling_files = [CONFIG_NAME, WEIGHTS_NAME] + modeling_files = [CONFIG_NAME, WEIGHTS_NAME, SAFE_WEIGHTS_NAME] for modeling_file in modeling_files: if os.path.isfile(os.path.join(checkpoint_folder, modeling_file)): shutil.copy(os.path.join(checkpoint_folder, modeling_file), os.path.join(output_dir, modeling_file)) diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py index 2a3c32673262..28387885de16 100644 --- a/src/transformers/training_args.py +++ b/src/transformers/training_args.py @@ -42,6 +42,7 @@ get_full_repo_name, is_accelerate_available, is_psutil_available, + is_safetensors_available, is_sagemaker_dp_enabled, is_sagemaker_mp_enabled, is_torch_available, @@ -261,6 +262,9 @@ class TrainingArguments: save_total_limit (`int`, *optional*): If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in `output_dir`. + save_safetensors (`bool`, *optional*, defaults to `False`): + Use [safetensors](https://huggingface.co/docs/safetensors) saving and loading for state dicts instead of + default `torch.load` and `torch.save`. save_on_each_node (`bool`, *optional*, defaults to `False`): When doing multi-node distributed training, whether to save models and checkpoints on each node, or only on the main one. @@ -720,6 +724,12 @@ class TrainingArguments: ) }, ) + save_safetensors: Optional[bool] = field( + default=False, + metadata={ + "help": "Use safetensors saving and loading for state dicts instead of default torch.load and torch.save." + }, + ) save_on_each_node: bool = field( default=False, metadata={ @@ -1166,6 +1176,17 @@ def __post_init__(self): f"steps, but found {self.save_steps}, which is not a round multiple of {self.eval_steps}." ) + safetensors_available = is_safetensors_available() + if self.save_safetensors and not safetensors_available: + raise ValueError(f"--save_safetensors={self.save_safetensors} requires safetensors to be installed!") + if not self.save_safetensors and safetensors_available: + logger.info( + f"Found safetensors installation, but --save_safetensors={self.save_safetensors}. " + f"Safetensors should be a preferred weights saving format due to security and performance reasons. " + f"If your model cannot be saved by safetensors please feel free to open an issue at " + f"https://github.com/huggingface/safetensors!" + ) + if self.load_best_model_at_end and self.metric_for_best_model is None: self.metric_for_best_model = "loss" if self.greater_is_better is None and self.metric_for_best_model is not None: diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 310842713bde..78b6afeacd4e 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -25,6 +25,7 @@ import tempfile import time import unittest +from itertools import product from pathlib import Path from unittest.mock import Mock, patch @@ -54,6 +55,7 @@ require_intel_extension_for_pytorch, require_optuna, require_ray, + require_safetensors, require_sentencepiece, require_sigopt, require_tokenizers, @@ -73,10 +75,13 @@ from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR from transformers.training_args import OptimizerNames from transformers.utils import ( + SAFE_WEIGHTS_INDEX_NAME, + SAFE_WEIGHTS_NAME, WEIGHTS_INDEX_NAME, WEIGHTS_NAME, is_apex_available, is_bitsandbytes_available, + is_safetensors_available, is_torchdistx_available, ) from transformers.utils.hp_naming import TrialShortNamer @@ -102,6 +107,9 @@ ) from transformers.modeling_utils import unwrap_model + if is_safetensors_available(): + import safetensors.torch + PATH_SAMPLE_TEXT = f"{get_tests_dir()}/fixtures/sample_text.txt" @@ -345,8 +353,9 @@ def get_regression_trainer(a=0, b=0, double_output=False, train_len=64, eval_len class TrainerIntegrationCommon: - def check_saved_checkpoints(self, output_dir, freq, total, is_pretrained=True): - file_list = [WEIGHTS_NAME, "training_args.bin", "optimizer.pt", "scheduler.pt", "trainer_state.json"] + def check_saved_checkpoints(self, output_dir, freq, total, is_pretrained=True, safe_weights=False): + weights_file = WEIGHTS_NAME if not safe_weights else SAFE_WEIGHTS_NAME + file_list = [weights_file, "training_args.bin", "optimizer.pt", "scheduler.pt", "trainer_state.json"] if is_pretrained: file_list.append("config.json") for step in range(freq, total, freq): @@ -356,7 +365,7 @@ def check_saved_checkpoints(self, output_dir, freq, total, is_pretrained=True): self.assertTrue(os.path.isfile(os.path.join(checkpoint, filename))) def check_best_model_has_been_loaded( - self, output_dir, freq, total, trainer, metric, greater_is_better=False, is_pretrained=True + self, output_dir, freq, total, trainer, metric, greater_is_better=False, is_pretrained=True, safe_weights=False ): checkpoint = os.path.join(output_dir, f"checkpoint-{(total // freq) * freq}") log_history = TrainerState.load_from_json(os.path.join(checkpoint, "trainer_state.json")).log_history @@ -370,7 +379,10 @@ def check_best_model_has_been_loaded( best_model.to(trainer.args.device) else: best_model = RegressionModel() - state_dict = torch.load(os.path.join(checkpoint, WEIGHTS_NAME)) + if not safe_weights: + state_dict = torch.load(os.path.join(checkpoint, WEIGHTS_NAME)) + else: + state_dict = safetensors.torch.load_file(os.path.join(checkpoint, SAFE_WEIGHTS_NAME)) best_model.load_state_dict(state_dict) best_model.to(trainer.args.device) self.assertTrue(torch.allclose(best_model.a, trainer.model.a)) @@ -394,24 +406,43 @@ def check_trainer_state_are_the_same(self, trainer_state, trainer_state1): _ = log1.pop(key, None) self.assertEqual(log, log1) - def convert_to_sharded_checkpoint(self, folder): + def convert_to_sharded_checkpoint(self, folder, save_safe=False, load_safe=False): # Converts a checkpoint of a regression model to a sharded checkpoint. - state_dict = torch.load(os.path.join(folder, WEIGHTS_NAME)) - os.remove(os.path.join(folder, WEIGHTS_NAME)) + if load_safe: + loader = safetensors.torch.load_file + weights_file = os.path.join(folder, SAFE_WEIGHTS_NAME) + else: + loader = torch.load + weights_file = os.path.join(folder, WEIGHTS_NAME) + + if save_safe: + extension = "safetensors" + saver = safetensors.torch.save_file + index_file = os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME) + shard_name = SAFE_WEIGHTS_NAME + else: + extension = "bin" + saver = torch.save + index_file = os.path.join(folder, WEIGHTS_INDEX_NAME) + shard_name = WEIGHTS_NAME + + state_dict = loader(weights_file) + + os.remove(weights_file) keys = list(state_dict.keys()) shard_files = [ - WEIGHTS_NAME.replace(".bin", f"-{idx+1:05d}-of-{len(keys):05d}.bin") for idx in range(len(keys)) + shard_name.replace(f".{extension}", f"-{idx+1:05d}-of-{len(keys):05d}.{extension}") + for idx in range(len(keys)) ] index = {"metadata": {}, "weight_map": {key: shard_files[i] for i, key in enumerate(keys)}} - save_index_file = os.path.join(folder, WEIGHTS_INDEX_NAME) - with open(save_index_file, "w", encoding="utf-8") as f: + with open(index_file, "w", encoding="utf-8") as f: content = json.dumps(index, indent=2, sort_keys=True) + "\n" f.write(content) for param_name, shard_file in zip(keys, shard_files): - torch.save({param_name: state_dict[param_name]}, os.path.join(folder, shard_file)) + saver({param_name: state_dict[param_name]}, os.path.join(folder, shard_file)) @require_torch @@ -1132,6 +1163,26 @@ def test_save_checkpoints(self): trainer.train() self.check_saved_checkpoints(tmpdir, 5, int(self.n_epochs * 64 / self.batch_size), False) + @require_safetensors + def test_safe_checkpoints(self): + for save_safetensors in [True, False]: + with tempfile.TemporaryDirectory() as tmpdir: + trainer = get_regression_trainer(output_dir=tmpdir, save_steps=5, save_safetensors=save_safetensors) + trainer.train() + self.check_saved_checkpoints( + tmpdir, 5, int(self.n_epochs * 64 / self.batch_size), safe_weights=save_safetensors + ) + + # With a regular model that is not a PreTrainedModel + with tempfile.TemporaryDirectory() as tmpdir: + trainer = get_regression_trainer( + output_dir=tmpdir, save_steps=5, pretrained=False, save_safetensors=save_safetensors + ) + trainer.train() + self.check_saved_checkpoints( + tmpdir, 5, int(self.n_epochs * 64 / self.batch_size), False, safe_weights=save_safetensors + ) + @require_torch_multi_gpu def test_run_seq2seq_double_train_wrap_once(self): # test that we don't wrap the model more than once @@ -1373,6 +1424,42 @@ def test_resume_training_with_shard_checkpoint(self): self.assertEqual(b, b1) self.check_trainer_state_are_the_same(state, state1) + @require_safetensors + @require_torch_up_to_2_gpus + def test_resume_training_with_safe_checkpoint(self): + # This test will fail for more than 2 GPUs since the batch size will get bigger and with the number of + # save_steps, the checkpoint will resume training at epoch 2 or more (so the data seen by the model + # won't be the same since the training dataloader is shuffled). + + for initial_safe in [False, True]: + for loaded_safe in [False, True]: + with tempfile.TemporaryDirectory() as tmpdir: + trainer = get_regression_trainer( + output_dir=tmpdir, + train_len=128, + save_steps=5, + learning_rate=0.1, + save_safetensors=initial_safe, + ) + trainer.train() + (a, b) = trainer.model.a.item(), trainer.model.b.item() + state = dataclasses.asdict(trainer.state) + + checkpoint = os.path.join(tmpdir, "checkpoint-5") + self.convert_to_sharded_checkpoint(checkpoint, load_safe=initial_safe, save_safe=loaded_safe) + + # Reinitialize trainer + trainer = get_regression_trainer( + output_dir=tmpdir, train_len=128, save_steps=5, learning_rate=0.1, save_safetensors=loaded_safe + ) + + trainer.train(resume_from_checkpoint=checkpoint) + (a1, b1) = trainer.model.a.item(), trainer.model.b.item() + state1 = dataclasses.asdict(trainer.state) + self.assertEqual(a, a1) + self.assertEqual(b, b1) + self.check_trainer_state_are_the_same(state, state1) + @require_torch_up_to_2_gpus def test_resume_training_with_gradient_accumulation(self): # This test will fail for more than 2 GPUs since the batch size will get bigger and with the number of @@ -1522,6 +1609,30 @@ def test_load_best_model_at_end(self): self.check_saved_checkpoints(tmpdir, 5, total, is_pretrained=False) self.check_best_model_has_been_loaded(tmpdir, 5, total, trainer, "eval_loss", is_pretrained=False) + @require_safetensors + def test_load_best_model_from_safetensors(self): + total = int(self.n_epochs * 64 / self.batch_size) + for save_safetensors, pretrained in product([False, True], [False, True]): + with tempfile.TemporaryDirectory() as tmpdir: + trainer = get_regression_trainer( + a=1.5, + b=2.5, + output_dir=tmpdir, + learning_rate=0.1, + eval_steps=5, + evaluation_strategy="steps", + save_steps=5, + load_best_model_at_end=True, + save_safetensors=save_safetensors, + pretrained=pretrained, + ) + self.assertFalse(trainer.args.greater_is_better) + trainer.train() + self.check_saved_checkpoints(tmpdir, 5, total, is_pretrained=pretrained, safe_weights=save_safetensors) + self.check_best_model_has_been_loaded( + tmpdir, 5, total, trainer, "eval_loss", is_pretrained=pretrained, safe_weights=save_safetensors + ) + @slow def test_trainer_eval_mrpc(self): MODEL_ID = "bert-base-cased-finetuned-mrpc" From 19ce467427d5dca92c6a75284a5b22b3ecbd8cf5 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 4 Apr 2023 09:20:13 -0400 Subject: [PATCH 268/277] Remove hack for dynamic modules and use Python functions instead (#22537) --- src/transformers/dynamic_module_utils.py | 57 +++++++++--------------- 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/src/transformers/dynamic_module_utils.py b/src/transformers/dynamic_module_utils.py index 26d877d80447..046980ad13b4 100644 --- a/src/transformers/dynamic_module_utils.py +++ b/src/transformers/dynamic_module_utils.py @@ -13,14 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. """Utilities to dynamically load objects from the Hub.""" - +import filecmp import importlib import os import re import shutil -import subprocess import sys -import tempfile from pathlib import Path from typing import Dict, Optional, Union @@ -45,6 +43,7 @@ def init_hf_modules(): init_path = Path(HF_MODULES_CACHE) / "__init__.py" if not init_path.exists(): init_path.touch() + importlib.invalidate_caches() def create_dynamic_module(name: Union[str, os.PathLike]): @@ -60,6 +59,7 @@ def create_dynamic_module(name: Union[str, os.PathLike]): init_path = dynamic_module_path / "__init__.py" if not init_path.exists(): init_path.touch() + importlib.invalidate_caches() def get_relative_imports(module_file): @@ -148,35 +148,9 @@ def get_class_in_module(class_name, module_path): """ Import a module on the cache directory for modules and extract a class from it. """ - with tempfile.TemporaryDirectory() as tmp_dir: - module_dir = Path(HF_MODULES_CACHE) / os.path.dirname(module_path) - module_file_name = module_path.split(os.path.sep)[-1] + ".py" - - # Copy to a temporary directory. We need to do this in another process to avoid strange and flaky error - # `ModuleNotFoundError: No module named 'transformers_modules.[module_dir_name].modeling'` - shutil.copy(f"{module_dir}/{module_file_name}", tmp_dir) - # On Windows, we need this character `r` before the path argument of `os.remove` - cmd = f'import os; os.remove(r"{module_dir}{os.path.sep}{module_file_name}")' - # We don't know which python binary file exists in an environment. For example, if `python3` exists but not - # `python`, the call `subprocess.run(["python", ...])` gives `FileNotFoundError` (about python binary). Notice - # that, if the file to be removed is not found, we also have `FileNotFoundError`, but it is not raised to the - # caller's process. - try: - subprocess.run(["python", "-c", cmd]) - except FileNotFoundError: - try: - subprocess.run(["python3", "-c", cmd]) - except FileNotFoundError: - pass - - # copy back the file that we want to import - shutil.copyfile(f"{tmp_dir}/{module_file_name}", f"{module_dir}/{module_file_name}") - - # import the module - module_path = module_path.replace(os.path.sep, ".") - module = importlib.import_module(module_path) - - return getattr(module, class_name) + module_path = module_path.replace(os.path.sep, ".") + module = importlib.import_module(module_path) + return getattr(module, class_name) def get_cached_module_file( @@ -273,13 +247,21 @@ def get_cached_module_file( create_dynamic_module(full_submodule) submodule_path = Path(HF_MODULES_CACHE) / full_submodule if submodule == pretrained_model_name_or_path.split(os.path.sep)[-1]: - # We always copy local files (we could hash the file to see if there was a change, and give them the name of - # that hash, to only copy when there is a modification but it seems overkill for now). - # The only reason we do the copy is to avoid putting too many folders in sys.path. - shutil.copy(resolved_module_file, submodule_path / module_file) + # We copy local files to avoid putting too many folders in sys.path. This copy is done when the file is new or + # has changed since last copy. + if not (submodule_path / module_file).exists() or not filecmp.cmp( + resolved_module_file, str(submodule_path / module_file) + ): + shutil.copy(resolved_module_file, submodule_path / module_file) + importlib.invalidate_caches() for module_needed in modules_needed: module_needed = f"{module_needed}.py" - shutil.copy(os.path.join(pretrained_model_name_or_path, module_needed), submodule_path / module_needed) + module_needed_file = os.path.join(pretrained_model_name_or_path, module_needed) + if not (submodule_path / module_needed).exists() or not filecmp.cmp( + module_needed_file, str(submodule_path / module_needed) + ): + shutil.copy(module_needed_file, submodule_path / module_needed) + importlib.invalidate_caches() else: # Get the commit hash # TODO: we will get this info in the etag soon, so retrieve it from there and not here. @@ -293,6 +275,7 @@ def get_cached_module_file( if not (submodule_path / module_file).exists(): shutil.copy(resolved_module_file, submodule_path / module_file) + importlib.invalidate_caches() # Make sure we also have every file with relative for module_needed in modules_needed: if not (submodule_path / module_needed).exists(): From 3a1e4b7901d1be58adc57d2316dbfae606f5f754 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 4 Apr 2023 15:26:45 +0200 Subject: [PATCH 269/277] [`bnb`] Fix typo (#22556) Update modeling_utils.py --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 27faa252788d..1e156ba9d157 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2606,7 +2606,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P if len(keys_on_cpu) > 0 and not load_in_8bit_fp32_cpu_offload: raise ValueError( "If you want to offload some keys to `cpu` or `disk`, you need to set " - "`load_in_8bit_fp32_cpu_offload=True`. Note that these modules will not be " + "`llm_int8_enable_fp32_cpu_offload=True`. Note that these modules will not be " " converted to 8-bit but kept in 32-bit." ) From 763a78e69620784c3babe6357e44109e166a689a Mon Sep 17 00:00:00 2001 From: Maziyar Panahi Date: Tue, 4 Apr 2023 15:28:57 +0200 Subject: [PATCH 270/277] Add id2label and label2id to model's config in run_xnil (#22558) Add id2label and label2id to config in run_xnil --- examples/pytorch/text-classification/run_xnli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/pytorch/text-classification/run_xnli.py b/examples/pytorch/text-classification/run_xnli.py index d26f59d0e314..871fa3013cf9 100755 --- a/examples/pytorch/text-classification/run_xnli.py +++ b/examples/pytorch/text-classification/run_xnli.py @@ -273,6 +273,8 @@ def main(): config = AutoConfig.from_pretrained( model_args.config_name if model_args.config_name else model_args.model_name_or_path, num_labels=num_labels, + id2label={str(i): label for i, label in enumerate(label_list)}, + label2id={label: i for i, label in enumerate(label_list)}, finetuning_task="xnli", cache_dir=model_args.cache_dir, revision=model_args.model_revision, From dad2f7fbc2d180a1fd22e21dbee8d86c6932a1e2 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 4 Apr 2023 16:21:57 +0200 Subject: [PATCH 271/277] Soft error whisper. (#22475) * Soft error whisper. * Fix format. --------- Co-authored-by: Ubuntu --- .../models/whisper/tokenization_whisper.py | 4 +-- ..._pipelines_automatic_speech_recognition.py | 33 ++++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/whisper/tokenization_whisper.py b/src/transformers/models/whisper/tokenization_whisper.py index 0160237304b6..24eb72a0b0f9 100644 --- a/src/transformers/models/whisper/tokenization_whisper.py +++ b/src/transformers/models/whisper/tokenization_whisper.py @@ -877,9 +877,7 @@ def new_chunk(): if previous_tokens: if return_timestamps: - # Last token should always be timestamps, so there shouldn't be - # leftover - raise ValueError( + logger.warning( "There was an error while processing timestamps, we haven't found a timestamp as last token. Was" " WhisperTimeStampLogitsProcessor used?" ) diff --git a/tests/pipelines/test_pipelines_automatic_speech_recognition.py b/tests/pipelines/test_pipelines_automatic_speech_recognition.py index 5db3e3e46c72..952508dca441 100644 --- a/tests/pipelines/test_pipelines_automatic_speech_recognition.py +++ b/tests/pipelines/test_pipelines_automatic_speech_recognition.py @@ -17,7 +17,7 @@ import numpy as np import pytest from datasets import load_dataset -from huggingface_hub import snapshot_download +from huggingface_hub import hf_hub_download, snapshot_download from transformers import ( MODEL_FOR_CTC_MAPPING, @@ -39,6 +39,7 @@ require_pyctcdecode, require_tf, require_torch, + require_torch_gpu, require_torchaudio, slow, ) @@ -1158,6 +1159,36 @@ def test_stride(self): output = speech_recognizer({"raw": waveform, "stride": (1000, 8000), "sampling_rate": 16_000}) self.assertEqual(output, {"text": "XB"}) + @slow + @require_torch_gpu + def test_slow_unfinished_sequence(self): + from transformers import GenerationConfig + + pipe = pipeline( + "automatic-speech-recognition", + model="vasista22/whisper-hindi-large-v2", + device="cuda:0", + ) + # Original model wasn't trained with timestamps and has incorrect generation config + pipe.model.generation_config = GenerationConfig.from_pretrained("openai/whisper-large-v2") + + audio = hf_hub_download("Narsil/asr_dummy", filename="hindi.ogg", repo_type="dataset") + + out = pipe( + audio, + return_timestamps=True, + ) + self.assertEqual( + out, + { + "chunks": [ + {"text": "", "timestamp": (18.94, 0.0)}, + {"text": "मिर्ची में कितने विभिन्न प्रजातियां हैं", "timestamp": (None, None)}, + ], + "text": "मिर्ची में कितने विभिन्न प्रजातियां हैं", + }, + ) + def require_ffmpeg(test_case): """ From 8b5ee45fd5a391aea407c5584cc82a268c2bdf5e Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 4 Apr 2023 16:05:22 +0100 Subject: [PATCH 272/277] Add TF port of BLIP (#22090) * Initial commit * more stash commit * Yet another stash commit * yet more stash commit * Mostly working except for docs / repo consistency * Stop importing model list from torch file * Add TF BLIP models to docs * Add auto classes * Move get_text_features and get_image_features * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip_text.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update tests/models/blip/test_modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update tests/models/blip/test_modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: Joao Gante * Update tests/models/blip/test_modeling_tf_blip_text.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip_text.py Co-authored-by: Joao Gante * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Use channels_last convolutions in TF (better performance + compatibility) * Remove _shape function * Move multi-line statement to one line in PT + TF * Specify tf.keras.layers instead of importing from it * Remove test_gradient_checkpointing and empty test_training methods * move some multi-line statements to one line * Update docstring for generate * Remove pruned heads set * Remove self.seq_len_dim * Fixed issues with loss computation, should resolve some tests. Also ensured that the PT version follows the config for output_attentions and output_hidden_states * ensure original model follows config in more cases * Skip the same cross-attention tests in the PT tests - didn't realize we did it twice! * Add training args throughout the models and layers * make fixup * Fix docstring for inputs_embeds * Add docstring for is_decoder * Add docstrings to text models * Remove redundant computation * Add unpack_inputs / keras_serializable * Add modeling_tf_blip to doctests * Add config classes for keras serialization * Changes to allow model porting with pt-to-tf * Quick fix to decoder head and test tweaks * Revert an issue with masking the embeddings outputs * Allow missing keys in some equivalence tests (for unused layers) * Add tf-pt equivalence tests back in * Update src/transformers/models/blip/modeling_tf_blip.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip_text.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * Update src/transformers/models/blip/modeling_tf_blip_text.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * make fixup * Refactor invert_attention_mask out into tf_utils * Re-enable cross-tests on the PT side too --------- Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Co-authored-by: Joao Gante Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- docs/source/en/index.mdx | 2 +- docs/source/en/model_doc/blip.mdx | 40 +- src/transformers/__init__.py | 22 + src/transformers/commands/pt_to_tf.py | 20 +- src/transformers/modeling_tf_utils.py | 33 + .../models/auto/modeling_tf_auto.py | 2 + src/transformers/models/blip/__init__.py | 42 +- src/transformers/models/blip/modeling_blip.py | 40 +- .../models/blip/modeling_tf_blip.py | 1753 +++++++++++++++++ .../models/blip/modeling_tf_blip_text.py | 1013 ++++++++++ .../models/blip_2/modeling_blip_2.py | 4 +- src/transformers/tf_utils.py | 28 + src/transformers/utils/dummy_tf_objects.py | 52 + tests/models/blip/test_modeling_blip.py | 6 + tests/models/blip/test_modeling_blip_text.py | 3 + tests/models/blip/test_modeling_tf_blip.py | 824 ++++++++ .../models/blip/test_modeling_tf_blip_text.py | 170 ++ tests/test_modeling_common.py | 18 +- tests/test_modeling_tf_common.py | 24 +- utils/check_repo.py | 7 + utils/documentation_tests.txt | 1 + 21 files changed, 4059 insertions(+), 45 deletions(-) create mode 100644 src/transformers/models/blip/modeling_tf_blip.py create mode 100644 src/transformers/models/blip/modeling_tf_blip_text.py create mode 100644 tests/models/blip/test_modeling_tf_blip.py create mode 100644 tests/models/blip/test_modeling_tf_blip_text.py diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 216425b29f62..8c73cc98dcf7 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -269,7 +269,7 @@ Flax), PyTorch, and/or TensorFlow. | BiT | ❌ | ❌ | ✅ | ❌ | ❌ | | Blenderbot | ✅ | ✅ | ✅ | ✅ | ✅ | | BlenderbotSmall | ✅ | ✅ | ✅ | ✅ | ✅ | -| BLIP | ❌ | ❌ | ✅ | ❌ | ❌ | +| BLIP | ❌ | ❌ | ✅ | ✅ | ❌ | | BLIP-2 | ❌ | ❌ | ✅ | ❌ | ❌ | | BLOOM | ❌ | ✅ | ✅ | ❌ | ❌ | | BridgeTower | ❌ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index 42116f48697e..12cc26f41826 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -1,4 +1,4 @@ -